Colm - yes, I just saw this and reverted the commit. I will comment in JIRA so that Endre can update the patch. Thanks.
> On Dec 7, 2017, at 11:04 AM, Colm O hEigeartaigh <[email protected]> wrote: > > > Hi Vel, > > This commit breaks the JDK7 compatibility (due to import > java.util.stream.Collectors;). > > Colm. > ---------- Forwarded message ---------- > From: <[email protected] <mailto:[email protected]>> > Date: Thu, Dec 7, 2017 at 1:31 PM > Subject: ranger git commit: RANGER-1827 microbenchmark for RangerPolicyEngine > To: [email protected] <mailto:[email protected]> > > > Repository: ranger > Updated Branches: > refs/heads/master 291bb3e9c -> c806d1777 > > > RANGER-1827 microbenchmark for RangerPolicyEngine > > Change-Id: Ie4cfd8e021a6e08ddcb1e6d58012f186c6732ccb > > Signed-off-by: Velmurugan Periasamy <[email protected] <mailto:[email protected]>> > > > Project: http://git-wip-us.apache.org/repos/asf/ranger/repo > <http://git-wip-us.apache.org/repos/asf/ranger/repo> > Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/c806d177 > <http://git-wip-us.apache.org/repos/asf/ranger/commit/c806d177> > Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/c806d177 > <http://git-wip-us.apache.org/repos/asf/ranger/tree/c806d177> > Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/c806d177 > <http://git-wip-us.apache.org/repos/asf/ranger/diff/c806d177> > > Branch: refs/heads/master > Commit: c806d17779144df3d0c60402a03cb167e69c1582 > Parents: 291bb3e > Author: Endre Zoltan Kovacs <[email protected] > <mailto:[email protected]>> > Authored: Tue Oct 3 14:44:18 2017 +0200 > Committer: Velmurugan Periasamy <[email protected] <mailto:[email protected]>> > Committed: Thu Dec 7 08:31:47 2017 -0500 > > ---------------------------------------------------------------------- > .../ranger/plugin/util/PerfDataRecorder.java | 43 +++- > ranger-tools/pom.xml | 21 ++ > .../ranger/policyengine/PerfTestClient.java | 32 +-- > .../RangerAccessRequestDeserializer.java | 52 ++++ > .../RangerResourceDeserializer.java | 50 ++++ > .../perftest/v2/RangerPolicyFactory.java | 256 +++++++++++++++++++ > .../RangerPolicyEnginePerformanceTest.java | 224 ++++++++++++++++ > .../src/test/resources/log4j.properties | 2 + > .../testdata/performance-chart.template | 44 ++++ > .../testdata/single-policy-template.json | 83 ++++++ > .../testdata/single-request-template.json | 16 ++ > 11 files changed, 788 insertions(+), 35 deletions(-) > ---------------------------------------------------------------------- > > > http://git-wip-us.apache.org/repos/asf/ranger/blob/c806d177/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java > > <http://git-wip-us.apache.org/repos/asf/ranger/blob/c806d177/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java> > ---------------------------------------------------------------------- > diff --git > a/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java > > b/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java > index 25f5334..2816369 100644 > --- > a/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java > +++ > b/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java > @@ -23,6 +23,8 @@ import org.apache.commons.collections.CollectionUtils; > import org.apache.commons.logging.Log; > import org.apache.commons.logging.LogFactory; > > +import com.google.common.collect.ImmutableMap; > + > import java.util.ArrayList; > import java.util.Collections; > import java.util.HashMap; > @@ -80,15 +82,15 @@ public class PerfDataRecorder { > long averageTimeSpent = 0L; > > if (perfStatistic.numberOfInvocations.get() != 0L) { > - averageTimeSpent = > perfStatistic.millisecondsSpent.get()/perfStatistic.numberOfInvocations.get(); > + averageTimeSpent = > perfStatistic.microSecondsSpent.get()/perfStatistic.numberOfInvocations.get(); > } > > String logMsg = "[" + tag + "]" + > - " execCount:" + > perfStatistic.numberOfInvocations.get() + > - ", totalTimeTaken:" + > perfStatistic.millisecondsSpent.get() + > - ", maxTimeTaken:" + > perfStatistic.maxTimeSpent.get() + > - ", minTimeTaken:" + > perfStatistic.minTimeSpent.get() + > - ", avgTimeTaken:" + averageTimeSpent; > + " execCount: " + > perfStatistic.numberOfInvocations.get() + > + ", totalTimeTaken: " + > perfStatistic.microSecondsSpent.get() + " μs" + > + ", maxTimeTaken: " + > perfStatistic.maxTimeSpent.get() + " μs" + > + ", minTimeTaken: " + > perfStatistic.minTimeSpent.get() + " μs" + > + ", avgTimeTaken: " + averageTimeSpent + " μs"; > > LOG.info(logMsg); > PERF.debug(logMsg); > @@ -125,15 +127,22 @@ public class PerfDataRecorder { > } > } > > - private static class PerfStatistic { > + public static Map<String, PerfStatistic> exposeStatistics() { > + if (instance != null) { > + return ImmutableMap.copyOf(instance.perfStatistics); > + } > + return ImmutableMap.of(); > + } > + > + public static class PerfStatistic { > private AtomicLong numberOfInvocations = new AtomicLong(0L); > - private AtomicLong millisecondsSpent = new AtomicLong(0L); > + private AtomicLong microSecondsSpent = new AtomicLong(0L); > private AtomicLong minTimeSpent = new > AtomicLong(Long.MAX_VALUE); > private AtomicLong maxTimeSpent = new > AtomicLong(Long.MIN_VALUE); > > void addPerfDataItem(final long timeTaken) { > numberOfInvocations.getAndIncrement(); > - millisecondsSpent.getAndAdd(timeTaken); > + microSecondsSpent.getAndAdd(timeTaken); > > long min = minTimeSpent.get(); > if(timeTaken < min) { > @@ -145,5 +154,21 @@ public class PerfDataRecorder { > maxTimeSpent.compareAndSet(max, timeTaken); > } > } > + > + public long getNumberOfInvocations() { > + return numberOfInvocations.get(); > + } > + > + public long getMicroSecondsSpent() { > + return microSecondsSpent.get(); > + } > + > + public long getMinTimeSpent() { > + return minTimeSpent.get(); > + } > + > + public long getMaxTimeSpent() { > + return maxTimeSpent.get(); > + } > } > } > > http://git-wip-us.apache.org/repos/asf/ranger/blob/c806d177/ranger-tools/pom.xml > > <http://git-wip-us.apache.org/repos/asf/ranger/blob/c806d177/ranger-tools/pom.xml> > ---------------------------------------------------------------------- > diff --git a/ranger-tools/pom.xml b/ranger-tools/pom.xml > index ff37fb3..c6db1f2 100644 > --- a/ranger-tools/pom.xml > +++ b/ranger-tools/pom.xml > @@ -58,5 +58,26 @@ > <artifactId>ranger-plugins-common</artifactId> > <version>${project.version}</version> > </dependency> > + <dependency> > + <groupId>be.ceau</groupId> > + <artifactId>chart</artifactId> > + <version>2.2.0</version> > + </dependency> > </dependencies> > + <build> > + <pluginManagement> > + <plugins> > + <plugin> > + <groupId>org.apache.maven.plugins</groupId> > + <artifactId>maven-surefire-plugin</artifactId> > + <version>2.20</version> > + <configuration> > + <excludes> > + > <exclude>**/RangerPolicyEnginePerformanceTest*</exclude> > + </excludes> > + </configuration> > + </plugin> > + </plugins> > + </pluginManagement> > + </build> > </project> > > http://git-wip-us.apache.org/repos/asf/ranger/blob/c806d177/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestClient.java > > <http://git-wip-us.apache.org/repos/asf/ranger/blob/c806d177/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestClient.java> > ---------------------------------------------------------------------- > diff --git > a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestClient.java > > b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestClient.java > index e6095cb..5bdb783 100644 > --- > a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestClient.java > +++ > b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestClient.java > @@ -42,14 +42,14 @@ public class PerfTestClient extends Thread { > final int maxCycles; > > List<RequestData> requests = null; > - static Gson gsonBuilder = null; > + private static Gson gson = null; > > static { > - > - gsonBuilder = new > GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z") > + GsonBuilder builder = new > GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z"); > + gson = builder > .setPrettyPrinting() > - > .registerTypeAdapter(RangerAccessRequest.class, new > RangerAccessRequestDeserializer()) > - > .registerTypeAdapter(RangerAccessResource.class, new > RangerResourceDeserializer()) > + > .registerTypeAdapter(RangerAccessRequest.class, new > RangerAccessRequestDeserializer(builder)) > + > .registerTypeAdapter(RangerAccessResource.class, new > RangerResourceDeserializer(builder)) > .create(); > } > > @@ -89,7 +89,7 @@ public class PerfTestClient extends Thread { > Type listType = new TypeToken<List<RequestData>>() { > }.getType(); > > - requests = gsonBuilder.fromJson(reader, listType); > + requests = gson.fromJson(reader, listType); > > ret = true; > } > @@ -156,24 +156,4 @@ public class PerfTestClient extends Thread { > public void setRequest(RangerAccessRequest request) { > this.request = request;} > public void setResult(RangerAccessResult result) { > this.result = result;} > } > - > - static class RangerAccessRequestDeserializer implements > JsonDeserializer<RangerAccessRequest> { > - @Override > - public RangerAccessRequest deserialize(JsonElement jsonObj, > Type type, > - > JsonDeserializationContext context) throws JsonParseException { > - RangerAccessRequestImpl ret = > gsonBuilder.fromJson(jsonObj, RangerAccessRequestImpl.class); > - > - ret.setAccessType(ret.getAccessType()); // to force > computation of isAccessTypeAny and isAccessTypeDelegatedAdmin > - > - return ret; > - } > - } > - > - static class RangerResourceDeserializer implements > JsonDeserializer<RangerAccessResource> { > - @Override > - public RangerAccessResource deserialize(JsonElement jsonObj, > Type type, > - > JsonDeserializationContext context) throws > JsonParseException { > - return gsonBuilder.fromJson(jsonObj, > RangerAccessResourceImpl.class); > - } > - } > } > > http://git-wip-us.apache.org/repos/asf/ranger/blob/c806d177/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerAccessRequestDeserializer.java > > <http://git-wip-us.apache.org/repos/asf/ranger/blob/c806d177/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerAccessRequestDeserializer.java> > ---------------------------------------------------------------------- > diff --git > a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerAccessRequestDeserializer.java > > b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerAccessRequestDeserializer.java > new file mode 100644 > index 0000000..fd7762d > --- /dev/null > +++ > b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerAccessRequestDeserializer.java > @@ -0,0 +1,52 @@ > +/* > + * Licensed to the Apache Software Foundation (ASF) under one > + * or more contributor license agreements. See the NOTICE file > + * distributed with this work for additional information > + * regarding copyright ownership. The ASF licenses this file > + * to you under the Apache License, Version 2.0 (the > + * "License"); you may not use this file except in compliance > + * with the License. You may obtain a copy of the License at > + * > + * http://www.apache.org/licenses/LICENSE-2.0 > <http://www.apache.org/licenses/LICENSE-2.0> > + * > + * Unless required by applicable law or agreed to in writing, > + * software distributed under the License is distributed on an > + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY > + * KIND, either express or implied. See the License for the > + * specific language governing permissions and limitations > + * under the License. > + */ > + > +package org.apache.ranger.policyengine; > + > +import java.lang.reflect.Type; > + > +import org.apache.ranger.plugin.policyengine.RangerAccessRequest; > +import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl; > + > +import com.google.gson.Gson; > +import com.google.gson.GsonBuilder; > +import com.google.gson.JsonDeserializationContext; > +import com.google.gson.JsonDeserializer; > +import com.google.gson.JsonElement; > +import com.google.gson.JsonParseException; > + > +/** > + * {@link JsonDeserializer} to assist {@link Gson} with selecting proper type > + * when encountering RangerAccessRequest interface in the source json. > + */ > +public class RangerAccessRequestDeserializer implements > JsonDeserializer<RangerAccessRequest> { > + > + private GsonBuilder gsonBuilder; > + > + public RangerAccessRequestDeserializer(final GsonBuilder builder) { > + this.gsonBuilder = builder; > + } > + > + @Override > + public RangerAccessRequest deserialize(JsonElement jsonObj, Type > type, JsonDeserializationContext context) throws JsonParseException { > + RangerAccessRequestImpl ret = > gsonBuilder.create().fromJson(jsonObj, RangerAccessRequestImpl.class); > + ret.setAccessType(ret.getAccessType()); // to force > computation of isAccessTypeAny and isAccessTypeDelegatedAdmin > + return ret; > + } > +} > \ No newline at end of file > > http://git-wip-us.apache.org/repos/asf/ranger/blob/c806d177/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerResourceDeserializer.java > > <http://git-wip-us.apache.org/repos/asf/ranger/blob/c806d177/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerResourceDeserializer.java> > ---------------------------------------------------------------------- > diff --git > a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerResourceDeserializer.java > > b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerResourceDeserializer.java > new file mode 100644 > index 0000000..da549f1 > --- /dev/null > +++ > b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerResourceDeserializer.java > @@ -0,0 +1,50 @@ > +/* > + * Licensed to the Apache Software Foundation (ASF) under one > + * or more contributor license agreements. See the NOTICE file > + * distributed with this work for additional information > + * regarding copyright ownership. The ASF licenses this file > + * to you under the Apache License, Version 2.0 (the > + * "License"); you may not use this file except in compliance > + * with the License. You may obtain a copy of the License at > + * > + * http://www.apache.org/licenses/LICENSE-2.0 > <http://www.apache.org/licenses/LICENSE-2.0> > + * > + * Unless required by applicable law or agreed to in writing, > + * software distributed under the License is distributed on an > + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY > + * KIND, either express or implied. See the License for the > + * specific language governing permissions and limitations > + * under the License. > + */ > + > +package org.apache.ranger.policyengine; > + > +import java.lang.reflect.Type; > + > +import org.apache.ranger.plugin.policyengine.RangerAccessResource; > +import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl; > + > +import com.google.gson.Gson; > +import com.google.gson.GsonBuilder; > +import com.google.gson.JsonDeserializationContext; > +import com.google.gson.JsonDeserializer; > +import com.google.gson.JsonElement; > +import com.google.gson.JsonParseException; > + > +/** > + * {@link JsonDeserializer} to assist {@link Gson} with selecting proper type > + * when encountering RangerAccessResource interface in the source json. > + */ > +public class RangerResourceDeserializer implements > JsonDeserializer<RangerAccessResource> { > + > + private GsonBuilder gsonBuilder; > + > + public RangerResourceDeserializer(final GsonBuilder builder) { > + this.gsonBuilder = builder; > + } > + > + @Override > + public RangerAccessResource deserialize(JsonElement jsonObj, Type > type, JsonDeserializationContext context) throws JsonParseException { > + return gsonBuilder.create().fromJson(jsonObj, > RangerAccessResourceImpl.class); > + } > +} > \ No newline at end of file > > http://git-wip-us.apache.org/repos/asf/ranger/blob/c806d177/ranger-tools/src/main/java/org/apache/ranger/policyengine/perftest/v2/RangerPolicyFactory.java > > <http://git-wip-us.apache.org/repos/asf/ranger/blob/c806d177/ranger-tools/src/main/java/org/apache/ranger/policyengine/perftest/v2/RangerPolicyFactory.java> > ---------------------------------------------------------------------- > diff --git > a/ranger-tools/src/main/java/org/apache/ranger/policyengine/perftest/v2/RangerPolicyFactory.java > > b/ranger-tools/src/main/java/org/apache/ranger/policyengine/perftest/v2/RangerPolicyFactory.java > new file mode 100644 > index 0000000..29b57d0 > --- /dev/null > +++ > b/ranger-tools/src/main/java/org/apache/ranger/policyengine/perftest/v2/RangerPolicyFactory.java > @@ -0,0 +1,256 @@ > +/* > + * Licensed to the Apache Software Foundation (ASF) under one > + * or more contributor license agreements. See the NOTICE file > + * distributed with this work for additional information > + * regarding copyright ownership. The ASF licenses this file > + * to you under the Apache License, Version 2.0 (the > + * "License"); you may not use this file except in compliance > + * with the License. You may obtain a copy of the License at > + * > + * http://www.apache.org/licenses/LICENSE-2.0 > <http://www.apache.org/licenses/LICENSE-2.0> > + * > + * Unless required by applicable law or agreed to in writing, > + * software distributed under the License is distributed on an > + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY > + * KIND, either express or implied. See the License for the > + * specific language governing permissions and limitations > + * under the License. > + */ > + > +package org.apache.ranger.policyengine.perftest.v2; > + > +import static com.google.common.base.Preconditions.checkState; > + > +import java.io.File; > +import java.lang.reflect.Type; > +import java.util.Arrays; > +import java.util.Collection; > +import java.util.List; > +import java.util.Map; > +import java.util.Random; > +import java.util.Set; > +import java.util.stream.Collectors; > + > +import org.apache.ranger.plugin.model.RangerPolicy; > +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem; > +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemAccess; > +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; > +import org.apache.ranger.plugin.policyengine.RangerAccessRequest; > +import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl; > +import org.apache.ranger.plugin.policyengine.RangerAccessResource; > +import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl; > +import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions; > +import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator; > +import org.apache.ranger.plugin.util.ServicePolicies; > +import org.apache.ranger.policyengine.RangerAccessRequestDeserializer; > +import org.apache.ranger.policyengine.RangerResourceDeserializer; > + > +import com.google.common.base.Charsets; > +import com.google.common.base.Function; > +import com.google.common.base.Throwables; > +import com.google.common.collect.ImmutableList; > +import com.google.common.collect.ImmutableMap; > +import com.google.common.collect.Iterables; > +import com.google.common.collect.Lists; > +import com.google.common.io.Files; > +import com.google.gson.Gson; > +import com.google.gson.GsonBuilder; > +import com.google.gson.reflect.TypeToken; > + > +/** > + * Factory for creating and wiring the object graph of {@link > ServicePolicies} and {@link RangerAccessRequest}. > + */ > +public class RangerPolicyFactory { > + > + private static final double SUCCESSFUL_ACCESS_RATE = 0.7d; > + > + private static final Random RANDOM = new Random(); > + > + private static final List<String> KNOWN_DATABASES = > createList("database", 10); > + > + private static final List<String> KNOWN_TABLES = createList("tables", > 100); > + > + private static final List<String> KNOWN_COLUMNS = > createList("column", 1000); > + > + private static final List<String> KNOWN_USERS = createList("user", > 1000); > + > + private static final List<String> RANDOM_VALUES = > createList("random", 100); > + > + private static final List<String> ALWAYS_ALLOWED_ACCESS_TYPES = > Arrays.asList("create", "select", "drop"); > + > + /** > + * Returns a {@link ServicePolicies service policy} instance with > containing the specified number of generated policies. > + * @param numberOfPolicies > + * @return > + */ > + public static ServicePolicies createServicePolicy(int > numberOfPolicies) { > + ServicePolicies servicePolicies = > loadTemplate("/testdata/test_servicepolicies_hive.json", new > TypeToken<ServicePolicies>(){}.getType()); > + mutate(servicePolicies, numberOfPolicies); > + return servicePolicies; > + } > + > + private static void mutate(ServicePolicies servicePolicies, int > numberOfPolicies) { > + servicePolicies.getPolicies().clear(); // reset > + servicePolicies.setPolicies(createPolicies(numberOfPolicies)); > + } > + > + private static List<RangerPolicy> createPolicies(int > numberOfPolicies) { > + List<RangerPolicy> policies = Lists.newArrayList(); > + String template = > readResourceFile("/testdata/single-policy-template.json"); > + for (int i = 0; i < numberOfPolicies; i++) { > + policies.add(createPolicyFromTemplate(template, i, > isAllowed())); > + } > + return policies; > + } > + > + private static RangerPolicy createPolicyFromTemplate(String template, > long policyId, boolean isAllowed) { > + RangerPolicy rangerPolicy = buildGson().fromJson(template, > RangerPolicy.class); > + rangerPolicy.setId(policyId); > + rangerPolicy.setName(String.format("generated policyname > #%s", policyId)); > + > rangerPolicy.setResources(createRangerPolicyResourceMap(isAllowed)); > + rangerPolicy.setPolicyItems(createPolicyItems(isAllowed)); > + return rangerPolicy; > + } > + > + private static Map<String, RangerPolicyResource> > createRangerPolicyResourceMap(boolean isAllowed) { > + RangerPolicyResource db = new RangerPolicyResource(isAllowed > ? pickFewRandomly(KNOWN_DATABASES): RANDOM_VALUES, false, false); > + RangerPolicyResource table = new > RangerPolicyResource(isAllowed ? pickFewRandomly(KNOWN_TABLES) : > RANDOM_VALUES, false, false); > + RangerPolicyResource column = new > RangerPolicyResource(isAllowed ? pickFewRandomly(KNOWN_COLUMNS) : > RANDOM_VALUES, false, false); > + return ImmutableMap.of("database", db, "table", table, > "column", column); > + } > + > + > + private static List<RangerPolicyItem> createPolicyItems(boolean > isAllowed) { > + List<RangerPolicyItem> policyItems = Lists.newArrayList(); > + for (int i = 0; i < 15; i++) { > + policyItems.add(createPolicyItem(isAllowed)); > + } > + return policyItems; > + } > + > + private static RangerPolicyItem createPolicyItem(boolean isAllowed) { > + RangerPolicyItem rangerPolicyItem = new RangerPolicyItem(); > + rangerPolicyItem.setDelegateAdmin(false); > + rangerPolicyItem.setUsers(isAllowed ? KNOWN_USERS : > RANDOM_VALUES); > + rangerPolicyItem.setAccesses(createAccesses(isAllowed)); > + return rangerPolicyItem; > + } > + > + private static List<RangerPolicyItemAccess> createAccesses(boolean > isAllowed) { > + List<RangerPolicyItemAccess> results = Lists.newArrayList(); > + results.addAll(Lists.transform(isAllowed ? > ALWAYS_ALLOWED_ACCESS_TYPES : RANDOM_VALUES, new Function<String, > RangerPolicyItemAccess>() { > + @Override > + public RangerPolicyItemAccess apply(String input) { > + return new RangerPolicyItemAccess(input, > true); > + } > + })); > + return results; > + } > + > + /** > + * Generates and returns a list of {@link RangerAccessRequest > requests} > + * @param nubmerOfRequests the number of requests to generate. > + * @return > + */ > + public static List<RangerAccessRequest> createAccessRequests(int > nubmerOfRequests) { > + List<RangerAccessRequest> result = Lists.newArrayList(); > + Gson gson = buildGson(); > + String template = > readResourceFile("/testdata/single-request-template.json"); > + for (int i = 0; i < nubmerOfRequests; i++) { > + RangerAccessRequestImpl accessRequest = > gson.fromJson(template, RangerAccessRequestImpl.class); > + result.add(mutate(accessRequest, isAllowed())); > + } > + return result; > + } > + > + /** > + * @return 10% of the time returns <code>true</code>, in which case > the generated request policy should evaluate to true. > + */ > + private static boolean isAllowed() { > + return RANDOM.nextDouble() < SUCCESSFUL_ACCESS_RATE; > + } > + > + private static RangerAccessRequest mutate(RangerAccessRequest > template, boolean shouldEvaluateToTrue) { > + RangerAccessRequestImpl accessRequest = > (RangerAccessRequestImpl) template; > + accessRequest.setResource(new > RangerAccessResourceImpl(createResourceElements(shouldEvaluateToTrue))); > + > accessRequest.setAccessType(pickOneRandomly(ALWAYS_ALLOWED_ACCESS_TYPES )); > + accessRequest.setRequestData(null); > + accessRequest.setUser(pickOneRandomly(KNOWN_USERS)); > + return accessRequest; > + } > + > + private static ImmutableMap<String, String> > createResourceElements(boolean shouldEvaluateToTrue) { > + String database = String.format("db_%s", System.nanoTime()); > + String table = String.format("table_%s", System.nanoTime()); > + String column = String.format("column_%s", System.nanoTime()); > + > + if (shouldEvaluateToTrue) { > + database = pickOneRandomly(KNOWN_DATABASES); > + table = pickOneRandomly(KNOWN_TABLES); > + column = pickOneRandomly(KNOWN_COLUMNS); > + } > + return ImmutableMap.of("database", database, "table", table, > "column", column); > + } > + > + private static List<String> createList(String name, int n) { > + List<String> results = Lists.newArrayList(); > + for (int i = 0; i < n; i++) { > + results.add(String.format("%s_%s",name, i)); > + } > + return results; > + } > + > + private static String pickOneRandomly(Collection<String> list) { > + return Iterables.get(list, RANDOM.nextInt(list.size())); > + } > + > + private static List<String> pickFewRandomly(final List<String> list) { > + int resultSize = RANDOM.nextInt(list.size()); > + > + Set<String> results = RANDOM.ints(resultSize, 0, > list.size()).boxed().map(new java.util.function.Function<Integer, String>() { > + @Override > + public String apply(Integer index) { > + return list.get(index); > + } > + }).collect(Collectors.<String>toSet()); > + > + return ImmutableList.copyOf(results); > + > + } > + > + private static <T> T loadTemplate(String file, Type targetType) { > + try { > + T model = > buildGson().fromJson(readResourceFile(file), targetType); > + return model; > + } catch (Exception e) { > + throw Throwables.propagate(e); > + } > + } > + > + private static Gson buildGson() { > + GsonBuilder gsonBuilder = new > GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z"); > + return gsonBuilder > + > .registerTypeAdapter(RangerAccessRequest.class, new > RangerAccessRequestDeserializer(gsonBuilder)) > + > .registerTypeAdapter(RangerAccessResource.class, new > RangerResourceDeserializer(gsonBuilder)) > + .setPrettyPrinting().create(); > + } > + > + public static String readResourceFile(String fileName) { > + try { > + File f = new > File(RangerPolicyFactory.class.getResource(fileName).toURI()); > + checkState(f.exists() && f.isFile() && f.canRead()); > + return Files.toString(f, Charsets.UTF_8); > + } catch (Exception e) { > + throw Throwables.propagate(e); > + } > + } > + > + public static RangerPolicyEngineOptions createPolicyEngineOption() { > + RangerPolicyEngineOptions policyEngineOptions = new > RangerPolicyEngineOptions(); > + policyEngineOptions.disableTagPolicyEvaluation = false; > + policyEngineOptions.evaluatorType = > RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED; > + policyEngineOptions.cacheAuditResults = false; > + policyEngineOptions.disableTrieLookupPrefilter = true; > + return policyEngineOptions; > + } > +} > > http://git-wip-us.apache.org/repos/asf/ranger/blob/c806d177/ranger-tools/src/test/java/org/apache/ranger/policyengine/RangerPolicyEnginePerformanceTest.java > > <http://git-wip-us.apache.org/repos/asf/ranger/blob/c806d177/ranger-tools/src/test/java/org/apache/ranger/policyengine/RangerPolicyEnginePerformanceTest.java> > ---------------------------------------------------------------------- > diff --git > a/ranger-tools/src/test/java/org/apache/ranger/policyengine/RangerPolicyEnginePerformanceTest.java > > b/ranger-tools/src/test/java/org/apache/ranger/policyengine/RangerPolicyEnginePerformanceTest.java > new file mode 100644 > index 0000000..82f5c1f > --- /dev/null > +++ > b/ranger-tools/src/test/java/org/apache/ranger/policyengine/RangerPolicyEnginePerformanceTest.java > @@ -0,0 +1,224 @@ > +/* > + * Licensed to the Apache Software Foundation (ASF) under one > + * or more contributor license agreements. See the NOTICE file > + * distributed with this work for additional information > + * regarding copyright ownership. The ASF licenses this file > + * to you under the Apache License, Version 2.0 (the > + * "License"); you may not use this file except in compliance > + * with the License. You may obtain a copy of the License at > + * > + * http://www.apache.org/licenses/LICENSE-2.0 > <http://www.apache.org/licenses/LICENSE-2.0> > + * > + * Unless required by applicable law or agreed to in writing, > + * software distributed under the License is distributed on an > + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY > + * KIND, either express or implied. See the License for the > + * specific language governing permissions and limitations > + * under the License. > + */ > + > +package org.apache.ranger.policyengine; > + > +import static com.google.common.collect.Iterables.get; > + > +import java.io.File; > +import java.io.IOException; > +import java.math.BigDecimal; > +import java.util.Arrays; > +import java.util.List; > +import java.util.Map; > +import java.util.Map.Entry; > +import java.util.Set; > +import java.util.concurrent.CountDownLatch; > +import java.util.stream.Collectors; > + > +import org.apache.commons.lang.text.StrSubstitutor; > +import org.apache.ranger.plugin.policyengine.RangerAccessRequest; > +import org.apache.ranger.plugin.policyengine.RangerPolicyEngineImpl; > +import org.apache.ranger.plugin.util.PerfDataRecorder; > +import org.apache.ranger.plugin.util.PerfDataRecorder.PerfStatistic; > +import org.apache.ranger.plugin.util.ServicePolicies; > +import org.apache.ranger.policyengine.perftest.v2.RangerPolicyFactory; > +import org.junit.After; > +import org.junit.AfterClass; > +import org.junit.Before; > +import org.junit.BeforeClass; > +import org.junit.Test; > +import org.junit.runner.RunWith; > +import org.junit.runners.Parameterized; > +import org.junit.runners.Parameterized.Parameter; > +import org.junit.runners.Parameterized.Parameters; > + > +import com.google.common.base.Charsets; > +import com.google.common.base.Function; > +import com.google.common.base.Splitter; > +import com.google.common.cache.CacheBuilder; > +import com.google.common.cache.CacheLoader; > +import com.google.common.cache.LoadingCache; > +import com.google.common.collect.ImmutableMap; > +import com.google.common.collect.Iterables; > +import com.google.common.collect.Lists; > +import com.google.common.collect.Sets; > +import com.google.common.collect.Table; > +import com.google.common.collect.TreeBasedTable; > +import com.google.common.io.Files; > + > +import be.ceau.chart.LineChart; > +import be.ceau.chart.color.Color; > +import be.ceau.chart.data.LineData; > +import be.ceau.chart.dataset.LineDataset; > + > +/** > + * A parameterized JUnit test that tests the performance of > RangerPolicyEngine, under increasing load of number of policies and > concurrent calls. > + * A cross product of the input parameters are generated and fed into the > test method. > + * This microbenchmark includes a warm-up phase so that any of the JIT > performance optimizations happen before the measurement of the policy > engine's performance. > + */ > +@RunWith(Parameterized.class) > +public class RangerPolicyEnginePerformanceTest { > + > + private static final String STATISTICS_KEY__ACCESS_ALLOWED = > "RangerPolicyEngine.isAccessAllowed"; > + > + /* pre-warming unit-under-test's method with this many call > iterations, so all possible JIT optimization happen before measuring > performance */ > + private static final int WARM_UP__ITERATIONS = 30_000; > + > + private static LoadingCache<Integer, List<RangerAccessRequest>> > requestsCache = > CacheBuilder.newBuilder().build(createAccessRequestsCacheLoader()); > + > + private static LoadingCache<Integer, ServicePolicies> > servicePoliciesCache = > CacheBuilder.newBuilder().build(createServicePoliciesCacheLoader()); > + > + @Parameter(0) > + public Integer numberOfPolicies; > + > + @Parameter(1) > + public Integer concurrency; > + > + /** > + * Generates a cross product of number-of-policies X concurrency > parameter sets. > + * @returns a collection of "tuples" (Object[]) of numberOfPolicies > and concurrency for the given test run > + */ > + @Parameters(name = "{index}: isAccessAllowed(policies: {0}, > concurrent calls: {1})") > + public static Iterable<Object[]> data() { > + // tree set for maintaining natural ordering > + Set<Integer> policies = Sets.newTreeSet(Lists.newArrayList(5, > 50, 100, 250, 500, 1_000, 2_000, 3_000, 4_000, 5_000)); > + Set<Integer> concurrency = > Sets.newTreeSet(Lists.newArrayList(1, 5, 10, 20, 30, 40, 50, 100)); > + > + return Iterables.transform(Sets.cartesianProduct(policies, > concurrency), new Function<List<Integer>, Object[]>() { > + @Override > + public Object[] apply(List<Integer> input) { > + return input.toArray(); > + } > + }); > + } > + > + @BeforeClass > + public static void init() throws IOException { > + PerfDataRecorder.initialize(Arrays.asList("")); // dummy > value initializes PerfDataRecorder > + > Files.write("policies;concurrency;average;min;max;total-time-spent;\n", > outputFile(), Charsets.UTF_8); > + } > + > + @AfterClass > + public static void chartResults() throws IOException { > + // row: policies > + // column: concurrency > + // value: average > + LineChart chart = buildChart(parsePerformanceTable()); > + String chartMarkup = > StrSubstitutor.replace(RangerPolicyFactory.readResourceFile("/testdata/performance-chart.template"), > ImmutableMap.of("data", chart.toJson())); > + Files.write(chartMarkup, new File("target", > "performance-chart.html"), Charsets.UTF_8); > + } > + > + @Before > + public void before() throws Exception { > + PerfDataRecorder.clearStatistics(); > + } > + > + @After > + public void after() throws IOException { > + Map<String, PerfStatistic> exposeStatistics = > PerfDataRecorder.exposeStatistics(); > + PerfStatistic stat = > exposeStatistics.get(STATISTICS_KEY__ACCESS_ALLOWED); > + long average = stat.getNumberOfInvocations() > 0 ? > (stat.getMicroSecondsSpent() / stat.getNumberOfInvocations()) : 0; > + Files.append(String.format("%s;%s;%s;%s;%s;%s;\n", > numberOfPolicies, concurrency, average, stat.getMinTimeSpent(), > stat.getMaxTimeSpent(), stat.getMicroSecondsSpent()), outputFile(), > Charsets.UTF_8); > + PerfDataRecorder.printStatistics(); > + PerfDataRecorder.clearStatistics(); > + } > + > + @Test > + public void policyEngineTest() throws InterruptedException { > + List<RangerAccessRequest> requests = > requestsCache.getUnchecked(concurrency); > + ServicePolicies servicePolicies = > servicePoliciesCache.getUnchecked(numberOfPolicies); > + > + final RangerPolicyEngineImpl rangerPolicyEngine = new > RangerPolicyEngineImpl("perf-test", servicePolicies, > RangerPolicyFactory.createPolicyEngineOption()); > + rangerPolicyEngine.preProcess(requests); > + > + for (int iterations = 0; iterations < WARM_UP__ITERATIONS; > iterations++) { > + // using return value of 'isAccessAllowed' with a > cheap operation: System#identityHashCode so JIT wont remove it as dead code > + > System.identityHashCode(rangerPolicyEngine.isAccessAllowed(requests.get(iterations > % concurrency), null)); > + PerfDataRecorder.clearStatistics(); > + } > + > + final CountDownLatch latch = new CountDownLatch(concurrency); > + for (int i = 0; i < concurrency; i++) { > + final RangerAccessRequest rangerAccessRequest = > requests.get(i); > + new Thread(new Runnable() { > + @Override > + public void run() { > + > System.identityHashCode(rangerPolicyEngine.isAccessAllowed(rangerAccessRequest, > null)); > + latch.countDown(); > + } > + }, String.format("Client #%s", i)).start(); > + } > + latch.await(); > + } > + > + private static File outputFile() { > + return new File("target", > "ranger-policy-engine-performance.csv"); > + } > + > + private static CacheLoader<Integer, List<RangerAccessRequest>> > createAccessRequestsCacheLoader() { > + return new CacheLoader<Integer, List<RangerAccessRequest>> () > { > + @Override > + public List<RangerAccessRequest> load(Integer > numberOfRequests) throws Exception { > + return > RangerPolicyFactory.createAccessRequests(numberOfRequests); > + } > + }; > + } > + > + private static CacheLoader<Integer, ServicePolicies> > createServicePoliciesCacheLoader() { > + return new CacheLoader<Integer, ServicePolicies>() { > + @Override > + public ServicePolicies load(Integer numberOfPolicies) > throws Exception { > + return > RangerPolicyFactory.createServicePolicy(numberOfPolicies); > + } > + }; > + } > + > + private static LineChart buildChart(Table<Long, Long, BigDecimal> > policyConcurrencyValueTable) { > + LineData lineData = new LineData(); > + LineChart chart = new LineChart(lineData); > + for (Entry<Long, Map<Long, BigDecimal>> concurrencyKeyedEntry > : policyConcurrencyValueTable.columnMap().entrySet()) { > + LineDataset dataset = new LineDataset() > + .setBackgroundColor(Color.TRANSPARENT) > + .setBorderColor(Color.random()) > + .setLabel(String.format("%s > client(s)", concurrencyKeyedEntry.getKey())) > + > .setData(concurrencyKeyedEntry.getValue().values()); > + lineData.addDataset(dataset); > + } > + > + for (Long policies : policyConcurrencyValueTable.rowKeySet()) > { > + lineData.addLabels(String.format("Policies %s", > policies)); > + } > + return chart; > + } > + > + private static Table<Long, Long, BigDecimal> parsePerformanceTable() > throws IOException { > + Table<Long, Long, BigDecimal> policyConcurrencyValueTable = > TreeBasedTable.create(); > + List<String> lines = Files.readLines(outputFile(), > Charsets.UTF_8).stream().skip(1).collect(Collectors.<String>toList()); > + Splitter splitter = Splitter.on(";"); > + for (String line : lines) { > + Iterable<String> values = splitter.split(line); > + Long policies = Long.valueOf(get(values, 0)); > + Long concurrency = Long.valueOf(get(values, 1)); > + BigDecimal averageValue = new BigDecimal(get(values, > 2)); > + policyConcurrencyValueTable.put(policies, > concurrency, averageValue); > + } > + return policyConcurrencyValueTable; > + } > +} > > http://git-wip-us.apache.org/repos/asf/ranger/blob/c806d177/ranger-tools/src/test/resources/log4j.properties > > <http://git-wip-us.apache.org/repos/asf/ranger/blob/c806d177/ranger-tools/src/test/resources/log4j.properties> > ---------------------------------------------------------------------- > diff --git a/ranger-tools/src/test/resources/log4j.properties > b/ranger-tools/src/test/resources/log4j.properties > index 4ea9d85..0bc879a 100644 > --- a/ranger-tools/src/test/resources/log4j.properties > +++ b/ranger-tools/src/test/resources/log4j.properties > @@ -42,6 +42,8 @@ ranger.perf.log.file=${java.io.tmpdir}/ranger-perf-test.log > > log4j.logger.org.apache.ranger.perf=${ranger.perf.logger} > log4j.additivity.org.apache.ranger.perf=false > +log4j.logger.org.apache.ranger.plugin.policyengine.RangerPolicyEngineImpl=OFF > +log4j.additivity.org.apache.ranger.plugin.policyengine.RangerPolicyEngineImpl=false > > log4j.appender.PERF=org.apache.log4j.DailyRollingFileAppender > log4j.appender.PERF.File=${ranger.perf.log.file} > > http://git-wip-us.apache.org/repos/asf/ranger/blob/c806d177/ranger-tools/src/test/resources/testdata/performance-chart.template > > <http://git-wip-us.apache.org/repos/asf/ranger/blob/c806d177/ranger-tools/src/test/resources/testdata/performance-chart.template> > ---------------------------------------------------------------------- > diff --git > a/ranger-tools/src/test/resources/testdata/performance-chart.template > b/ranger-tools/src/test/resources/testdata/performance-chart.template > new file mode 100644 > index 0000000..0366fb6 > --- /dev/null > +++ b/ranger-tools/src/test/resources/testdata/performance-chart.template > @@ -0,0 +1,44 @@ > +{{!-- > + Licensed to the Apache Software Foundation (ASF) under one or more > + contributor license agreements. See the NOTICE file distributed with > + this work for additional information regarding copyright ownership. > + The ASF licenses this file to You under the Apache License, Version 2.0 > + (the "License"); you may not use this file except in compliance with > + the License. You may obtain a copy of the License at > + > + http://www.apache.org/licenses/LICENSE-2.0 > <http://www.apache.org/licenses/LICENSE-2.0> > + > + Unless required by applicable law or agreed to in writing, software > + distributed under the License is distributed on an "AS IS" BASIS, > + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. > + See the License for the specific language governing permissions and > + limitations under the License. > +--}} > +<!doctype html> > +<html> > +<head> > + <title>Performance Chart</title> > + <script > src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.4.0/Chart.min.js > <https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.4.0/Chart.min.js>"></script> > + <style> > + canvas{ > + -moz-user-select: none; > + -webkit-user-select: none; > + -ms-user-select: none; > + } > + </style> > +</head> > + > +<body> > + <div style="width:90%;"> > + <canvas id="canvas"></canvas> > + </div> > + <script> > + var config = ${data}; > + > + window.onload = function() { > + var ctx = document.getElementById("canvas").getContext("2d"); > + window.myLine = new Chart(ctx, config); > + }; > + </script> > +</body> > +</html> > \ No newline at end of file > > http://git-wip-us.apache.org/repos/asf/ranger/blob/c806d177/ranger-tools/src/test/resources/testdata/single-policy-template.json > > <http://git-wip-us.apache.org/repos/asf/ranger/blob/c806d177/ranger-tools/src/test/resources/testdata/single-policy-template.json> > ---------------------------------------------------------------------- > diff --git > a/ranger-tools/src/test/resources/testdata/single-policy-template.json > b/ranger-tools/src/test/resources/testdata/single-policy-template.json > new file mode 100644 > index 0000000..c030803 > --- /dev/null > +++ b/ranger-tools/src/test/resources/testdata/single-policy-template.json > @@ -0,0 +1,83 @@ > +{ > + "service": "cl1_hive", > + "name": "cl1_hive-1-20151212014502", > + "isAuditEnabled": false, > + "resources": { > + "database": { > + "values": [ > + "*" > + ], > + "isExcludes": false, > + "isRecursive": false > + }, > + "column": { > + "values": [ > + "*" > + ], > + "isExcludes": false, > + "isRecursive": false > + }, > + "table": { > + "values": [ > + "*" > + ], > + "isExcludes": false, > + "isRecursive": false > + } > + }, > + "policyItems": [ > + { > + "accesses": [ > + { > + "type": "select", > + "isAllowed": true > + }, > + { > + "type": "update", > + "isAllowed": true > + }, > + { > + "type": "create", > + "isAllowed": true > + }, > + { > + "type": "drop", > + "isAllowed": true > + }, > + { > + "type": "alter", > + "isAllowed": true > + }, > + { > + "type": "index", > + "isAllowed": true > + }, > + { > + "type": "lock", > + "isAllowed": true > + }, > + { > + "type": "all", > + "isAllowed": true > + } > + ], > + "users": [ > + "ambari-qa" > + ], > + "groups": [ > + ], > + "conditions": [ > + ], > + "delegateAdmin": true, > + "isEnabled": true > + } > + ], > + "denyPolicyItems": [ > + ], > + "allowExceptions": [ > + ], > + "denyExceptions": [ > + ], > + "id": 2, > + "isEnabled": true > +} > > http://git-wip-us.apache.org/repos/asf/ranger/blob/c806d177/ranger-tools/src/test/resources/testdata/single-request-template.json > > <http://git-wip-us.apache.org/repos/asf/ranger/blob/c806d177/ranger-tools/src/test/resources/testdata/single-request-template.json> > ---------------------------------------------------------------------- > diff --git > a/ranger-tools/src/test/resources/testdata/single-request-template.json > b/ranger-tools/src/test/resources/testdata/single-request-template.json > new file mode 100644 > index 0000000..1bc409d > --- /dev/null > +++ b/ranger-tools/src/test/resources/testdata/single-request-template.json > @@ -0,0 +1,16 @@ > +{ > + "request": { > + "resource": { > + "elements": { > + "database": "default", > + "table": "tbl-0", > + "column": "col-2" > + } > + }, > + "accessType": "select", > + "user": "hrt_1", > + "userGroups": [ > + ], > + "requestData": "use default" > + } > +} > > > > > -- > Colm O hEigeartaigh > > Talend Community Coder > http://coders.talend.com <http://coders.talend.com/>
