Repository: incubator-ranger Updated Branches: refs/heads/tag-policy 8df9b1014 -> 31a65f48a
RANGER-596: Added Geolocation support Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/31a65f48 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/31a65f48 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/31a65f48 Branch: refs/heads/tag-policy Commit: 31a65f48a7f67c351e16abc4e606b37bcadfaca7 Parents: 8df9b10 Author: Abhay Kulkarni <[email protected]> Authored: Wed Jul 8 16:48:45 2015 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Fri Aug 7 18:24:32 2015 -0700 ---------------------------------------------------------------------- .../admin/client/RangerAdminRESTClient.java | 1 + .../RangerScriptExecutionContext.java | 9 + .../RangerAbstractGeolocationProvider.java | 178 +++++++++++++ .../RangerFileBasedGeolocationProvider.java | 35 +++ .../ranger/plugin/geo/BinarySearchTree.java | 222 ++++++++++++++++ .../ranger/plugin/geo/GeolocationMetadata.java | 101 ++++++++ .../apache/ranger/plugin/geo/RangeChecker.java | 24 ++ .../plugin/geo/RangerGeolocationData.java | 194 ++++++++++++++ .../plugin/geo/RangerGeolocationDatabase.java | 101 ++++++++ .../apache/ranger/plugin/geo/ValuePrinter.java | 91 +++++++ .../ranger/plugin/geo/ValueProcessor.java | 24 ++ .../ranger/plugin/store/GeolocationStore.java | 32 +++ .../plugin/store/file/GeolocationFileStore.java | 256 +++++++++++++++++++ .../plugin/policyengine/TestPolicyEngine.java | 14 +- agents-common/src/test/resources/geo/geo.txt | 29 +++ .../policyengine/test_policyengine_hdfs.json | 57 ++++- .../client/RangerAdminJersey2RESTClient.java | 1 - pom.xml | 2 +- 18 files changed, 1360 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/31a65f48/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java index 6739753..8a3f9ba 100644 --- a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java +++ b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java @@ -24,6 +24,7 @@ import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.GenericType; import com.sun.jersey.api.client.WebResource; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.security.AccessControlException; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/31a65f48/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptExecutionContext.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptExecutionContext.java b/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptExecutionContext.java index 803f4bf..6c77d80 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptExecutionContext.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptExecutionContext.java @@ -300,4 +300,13 @@ public final class RangerScriptExecutionContext { } return ret; } + + public final String getGeolocation(String attributeName) { + String ret = null; + + if (StringUtils.isNotBlank(attributeName)) { + ret = (String) getEvaluationContext().get(attributeName); + } + return ret; + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/31a65f48/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAbstractGeolocationProvider.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAbstractGeolocationProvider.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAbstractGeolocationProvider.java new file mode 100644 index 0000000..3f52001 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAbstractGeolocationProvider.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.contextenricher; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonParseException; +import com.google.gson.JsonSyntaxException; +import com.google.gson.reflect.TypeToken; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.geo.RangerGeolocationData; +import org.apache.ranger.plugin.geo.RangerGeolocationDatabase; +import org.apache.ranger.plugin.policyengine.RangerAccessRequest; +import org.apache.ranger.plugin.store.GeolocationStore; + +import java.lang.reflect.Type; +import java.util.Map; + +public abstract class RangerAbstractGeolocationProvider extends RangerAbstractContextEnricher { + + private static final Log LOG = LogFactory.getLog(RangerAbstractGeolocationProvider.class); + + public static final String ENRICHER_OPTION_GEOLOCATION_SOURCE_LOADER_OPTIONS = "geolocation.source.loader.options"; + public static final String ENRICHER_OPTION_GEOLOCATION_META_PREFIX = "geolocation.meta.prefix"; + + public static final String KEY_CONTEXT_GEOLOCATION_PREFIX = "LOCATION_"; + private GeolocationStore store = null; + private String geoMetaPrefix; + + abstract public String getGeoSourceLoader(); + + @Override + public void init() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAbstractGeolocationProvider.init()"); + } + + super.init(); + + geoMetaPrefix = getOption(ENRICHER_OPTION_GEOLOCATION_META_PREFIX); + if (geoMetaPrefix == null) { + geoMetaPrefix = ""; + } + + String geoSourceLoader = getGeoSourceLoader(); + + String geoSourceLoaderOptions = getOption(ENRICHER_OPTION_GEOLOCATION_SOURCE_LOADER_OPTIONS); + if (StringUtils.isBlank(geoSourceLoaderOptions)) { + geoSourceLoaderOptions = "{}"; + } + + Map<String, String> context = null; + GeolocationStore geoStore = null; + + + try { + Gson gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z") + .setPrettyPrinting() + .create(); + + Type mapType = new TypeToken<Map<String, String>>() {}.getType(); + context = gsonBuilder.fromJson(geoSourceLoaderOptions, mapType); + + } catch (JsonSyntaxException exception) { + LOG.error("RangerAbstractGeolocationProvider.init() - Cannot initialize geolocation.source.loader.options map, valueString=" + + geoSourceLoaderOptions + ", exception=" + exception); + } catch (JsonParseException exception) { + LOG.error("RangerAbstractGeolocationProvider.init() - Cannot initilize geolocation.source.loader.options map, valueString=" + + geoSourceLoaderOptions + ", exception=" + exception); + } + + if (context != null) { + try { + // Get the class definition and ensure it is of the correct type + @SuppressWarnings("unchecked") + Class<GeolocationStore> geoSourceLoaderClass = (Class<GeolocationStore>) Class.forName(geoSourceLoader); + // instantiate the loader class and initialize it with options + geoStore = geoSourceLoaderClass.newInstance(); + } + catch (ClassNotFoundException exception) { + LOG.error("RangerAbstractGeolocationProvider.init() - Class " + geoSourceLoader + " not found, exception=" + exception); + } + catch (ClassCastException exception) { + LOG.error("RangerAbstractGeolocationProvider.init() - Class " + geoSourceLoader + " is not a type of GeolocationStore, exception=" + exception); + } + catch (IllegalAccessException exception) { + LOG.error("RangerAbstractGeolocationProvider.init() - Class " + geoSourceLoader + " could not be instantiated, exception=" + exception); + } + catch (InstantiationException exception) { + LOG.error("RangerAbstractGeolocationProvider.init() - Class " + geoSourceLoader + " could not be instantiated, exception=" + exception); + } + + if (geoStore != null) { + try { + geoStore.init(context); + store = geoStore; + } catch (Exception exception) { + LOG.error("RangerAbstractGeolocationProvider.init() - geoLocation Store cannot be initialized, exception=" + exception); + } + } + } + + if (store == null) { + LOG.error("RangerAbstractGeolocationProvider.init() - is not initialized correctly."); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAbstractGeolocationProvider.init()"); + } + } + + @Override + public void enrich(RangerAccessRequest request) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAbstractGeolocationProvider.enrich(" + request + ")"); + } + + RangerGeolocationData geolocation = null; + + String clientIPAddress = request.getClientIPAddress(); + + if (LOG.isDebugEnabled()) { + LOG.debug("RangerAbstractGeolocationProvider.enrich() - clientIPAddress=" + clientIPAddress); + } + + if (StringUtils.isNotBlank(clientIPAddress) && store != null) { + + geolocation = store.getGeoLocation(clientIPAddress); + + if (geolocation != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerAbstractGeolocationProvider.enrich() - Country=" + geolocation.toString()); + } + Map<String, Object> context = request.getContext(); + + String[] geoAttrValues = geolocation.getLocationData(); + + RangerGeolocationDatabase database = store.getGeoDatabase(); + String[] attributeNames = database.getMetadata().getLocationDataItemNames(); + + for (int i = 0; i < geoAttrValues.length && i < attributeNames.length; i++) { + String contextName = KEY_CONTEXT_GEOLOCATION_PREFIX + geoMetaPrefix + attributeNames[i]; + context.put(contextName, geoAttrValues[i]); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerAbstractGeolocationProvider.enrich() - clientIPAddress '" + clientIPAddress + "' not found."); + } + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerAbstractGeolocationProvider.enrich() - clientIPAddress is null or blank, cannot get geolocation"); + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAbstractGeolocationProvider.enrich(" + request + ")"); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/31a65f48/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedGeolocationProvider.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedGeolocationProvider.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedGeolocationProvider.java new file mode 100644 index 0000000..ea599c7 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedGeolocationProvider.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.contextenricher; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class RangerFileBasedGeolocationProvider extends RangerAbstractGeolocationProvider { + + private static final Log LOG = LogFactory.getLog(RangerFileBasedGeolocationProvider.class); + + public static final String GEOLOCATION_SOURCE_LOADER_FILELOADER = "org.apache.ranger.plugin.store.file.GeolocationFileStore"; + + @Override + public String getGeoSourceLoader() { + return GEOLOCATION_SOURCE_LOADER_FILELOADER; + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/31a65f48/agents-common/src/main/java/org/apache/ranger/plugin/geo/BinarySearchTree.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/geo/BinarySearchTree.java b/agents-common/src/main/java/org/apache/ranger/plugin/geo/BinarySearchTree.java new file mode 100644 index 0000000..8bee24e --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/geo/BinarySearchTree.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.geo; + +public class BinarySearchTree<T extends Comparable<T> & RangeChecker<V>, V> { + private Node<T> root; + private int size = 0; + + public BinarySearchTree() { + root = null; + } + + public void insert(final T value) { + Node<T> node = new Node<T>(value); + + if (root == null) { + root = node; + return; + } + + Node<T> focusNode = root; + Node<T> parent; + + while (true) { + parent = focusNode; + + int comparison = focusNode.getValue().compareTo(value); + if (comparison == 0) { + return; + } + if (comparison < 0) { + focusNode = focusNode.getRight(); + if (focusNode == null) { + parent.setRight(node); + return; + } + } else { + focusNode = focusNode.getLeft(); + if (focusNode == null) { + parent.setLeft(node); + return; + } + } + } + } + + public T find(final V value) { + Node<T> focusNode = root; + + int rangeCheck; + + while (focusNode != null) { + rangeCheck = focusNode.getValue().compareToRange(value); + if (rangeCheck == 0) { + break; + } else if (rangeCheck < 0) { + focusNode = focusNode.getRight(); + } else { + focusNode = focusNode.getLeft(); + } + } + + return focusNode == null ? null : focusNode.getValue(); + } + + final public void preOrderTraverseTree(final ValueProcessor<T> processor) { + preOrderTraverseTree(getRoot(), processor); + } + + final public void inOrderTraverseTree(final ValueProcessor<T> processor) { + inOrderTraverseTree(getRoot(), processor); + } + + Node<T> getRoot() { + return root; + } + + void setRoot(final Node<T> newRoot) { + root = newRoot; + } + + void rebalance() { + Node<T> dummy = new Node<T>(null); + dummy.setRight(root); + + setRoot(dummy); + + degenerate(); + reconstruct(); + + setRoot(getRoot().getRight()); + } + + final void inOrderTraverseTree(final Node<T> focusNode, final ValueProcessor<T> processor) { + if (focusNode != null) { + inOrderTraverseTree(focusNode.getLeft(), processor); + processor.process(focusNode.getValue()); + inOrderTraverseTree(focusNode.getRight(), processor); + } + } + + final void preOrderTraverseTree(final Node<T> focusNode, final ValueProcessor<T> processor) { + if (focusNode != null) { + processor.process(focusNode.getValue()); + + preOrderTraverseTree(focusNode.getLeft(), processor); + preOrderTraverseTree(focusNode.getRight(), processor); + } + } + + private void degenerate() { + + Node<T> remainder, temp, sentinel; + + sentinel = getRoot(); + remainder = sentinel.getRight(); + + size = 0; + + while (remainder != null) { + if (remainder.getLeft() == null) { + sentinel = remainder; + remainder = remainder.getRight(); + size++; + } else { + temp = remainder.getLeft(); + remainder.setLeft(temp.getRight()); + temp.setRight(remainder); + remainder = temp; + sentinel.setRight(temp); + } + } + } + + private void reconstruct() { + + int sz = size; + Node<T> node = getRoot(); + + int fullCount = fullSize(sz); + rotateLeft(node, sz - fullCount); + + for (sz = fullCount; sz > 1; sz /= 2) { + rotateLeft(node, sz / 2); + } + } + + private void rotateLeft(Node<T> root, final int count) { + if (root == null) return; + + Node<T> scanner = root; + + for (int i = 0; i < count; i++) { + //Leftward rotation + Node<T> child = scanner.getRight(); + scanner.setRight(child.getRight()); + scanner = child.getRight(); + child.setRight(scanner.getLeft()); + scanner.setLeft(child); + } + } + + private int fullSize(final int size) { + // Full portion of a complete tree + int ret = 1; + while (ret <= size) { + ret = 2*ret + 1; + } + return ret / 2; + } + + static class Node<T> { + private T value; + private Node<T> left; + private Node<T> right; + + public Node(final T value) { + this.value = value; + } + + public T getValue() { + return value; + } + + public void setValue(final T value) { + this.value = value; + } + + public Node<T> getLeft() { + return left; + } + + public void setLeft(final Node<T> left) { + this.left = left; + } + + public Node<T> getRight() { + return right; + } + + public void setRight(final Node<T> right) { + this.right = right; + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/31a65f48/agents-common/src/main/java/org/apache/ranger/plugin/geo/GeolocationMetadata.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/geo/GeolocationMetadata.java b/agents-common/src/main/java/org/apache/ranger/plugin/geo/GeolocationMetadata.java new file mode 100644 index 0000000..02f4940 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/geo/GeolocationMetadata.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.geo; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class GeolocationMetadata { + private static final Log LOG = LogFactory.getLog(GeolocationMetadata.class); + + private String[] locationDataItemNames = new String[0]; + + public static GeolocationMetadata create(String fields[], int index) { + GeolocationMetadata ret = null; + + if (fields.length > 2) { + String[] metadataNames = new String[fields.length-2]; + + for (int i = 2; i < fields.length; i++) { + metadataNames[i-2] = fields[i]; + } + ret = new GeolocationMetadata(metadataNames); + } else { + LOG.error("GeolocationMetadata.createMetadata() - Not enough fields specified, need {start, end, location} at " + index); + } + + return ret; + } + + GeolocationMetadata() {} + + GeolocationMetadata(final String[] locationDataItemNames) { + this.locationDataItemNames = locationDataItemNames; + } + + public int getDataItemNameIndex(final String dataItemName) { + int ret = -1; + + if (!StringUtils.isBlank(dataItemName)) { + for (int i = 0; i < locationDataItemNames.length; i++) { + if (locationDataItemNames[i].equals(dataItemName)) { + ret = i; + break; + } + } + } + + return ret; + } + + public String[] getLocationDataItemNames() { + return locationDataItemNames; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toStringDump(sb); + + return sb.toString(); + } + + private StringBuilder toString(StringBuilder sb) { + sb.append("locationDataItems={"); + + for (int i = 0; i < locationDataItemNames.length; i++) { + sb.append(locationDataItemNames[i]).append(", "); + } + + sb.append("}"); + return sb; + } + private StringBuilder toStringDump(StringBuilder sb) { + sb.append("FROM_IP,TO_IP,"); + + for (int i = 0; i < locationDataItemNames.length; i++) { + sb.append(locationDataItemNames[i]).append(", "); + } + + return sb; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/31a65f48/agents-common/src/main/java/org/apache/ranger/plugin/geo/RangeChecker.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/geo/RangeChecker.java b/agents-common/src/main/java/org/apache/ranger/plugin/geo/RangeChecker.java new file mode 100644 index 0000000..576a136 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/geo/RangeChecker.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.geo; + +public interface RangeChecker<V> { + int compareToRange(V value); +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/31a65f48/agents-common/src/main/java/org/apache/ranger/plugin/geo/RangerGeolocationData.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/geo/RangerGeolocationData.java b/agents-common/src/main/java/org/apache/ranger/plugin/geo/RangerGeolocationData.java new file mode 100644 index 0000000..9cc1a3f --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/geo/RangerGeolocationData.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.geo; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.net.InetAddress; +import java.net.UnknownHostException; + +public class RangerGeolocationData implements Comparable<RangerGeolocationData>, RangeChecker<Long> { + private static final Log LOG = LogFactory.getLog(RangerGeolocationData.class); + + private static final Character IPSegmentsSeparator = '.'; + + private long fromIPAddress; + private long toIPAddress; + private String[] locationData = new String[0]; + + public static RangerGeolocationData create(String fields[], int index, boolean useDotFormat) { + + RangerGeolocationData data = null; + + if (fields.length > 2) { + String startAddress = fields[0]; + String endAddress = fields[1]; + + if (RangerGeolocationData.validateAsIP(startAddress, useDotFormat) && RangerGeolocationData.validateAsIP(endAddress, useDotFormat)) { + + long startIP, endIP; + if (useDotFormat) { + startIP = RangerGeolocationData.ipAddressToLong(startAddress); + endIP = RangerGeolocationData.ipAddressToLong(endAddress); + } else { + startIP = Long.valueOf(startAddress); + endIP = Long.valueOf(endAddress); + } + + if ((endIP - startIP) >= 0) { + + String[] locationData = new String[fields.length-2]; + for (int i = 2; i < fields.length; i++) { + locationData[i-2] = fields[i]; + } + data = new RangerGeolocationData(startIP, endIP, locationData); + } + } + + } else { + LOG.error("GeolocationMetadata.createMetadata() - Not enough fields specified, need {start, end, location} at " + index); + } + return data; + } + + public RangerGeolocationData(final long fromIPAddress, final long toIPAddress, final String[] locationData) { + this.fromIPAddress = fromIPAddress; + this.toIPAddress = toIPAddress; + this.locationData = locationData; + } + + public String[] getLocationData() { + return locationData; + } + + @Override + public int compareTo(final RangerGeolocationData other) { + int ret = Long.compare(fromIPAddress, other.fromIPAddress); + if (ret == 0) { + ret = Long.compare(toIPAddress, other.toIPAddress); + } + return ret; + } + + @Override + public int compareToRange(final Long ip) { + int ret = Long.compare(fromIPAddress, ip.longValue()); + + if (ret < 0) { + ret = Long.compare(toIPAddress, ip.longValue()); + if (ret > 0) { + ret = 0; + } + } + + return ret; + } + + public static long ipAddressToLong(final String ipAddress) { + + long ret = 0L; + + try { + byte[] bytes = InetAddress.getByName(ipAddress).getAddress(); + + if (bytes != null && bytes.length <= 4) { + for (int i = 0; i < bytes.length; i++) { + int val = bytes[i] < 0 ? (256 + bytes[i]) : bytes[i]; + ret += (val << (8 * (3 - i))); + } + } + } + catch (UnknownHostException exception) { + LOG.error("RangerGeolocationData.ipAddressToLong() - Invalid IP address " + ipAddress); + } + + return ret; + } + + public static String unsignedIntToIPAddress(final long val) { + if (val <= 0) { + return ""; + } + long remaining = val; + String segments[] = new String[4]; + for (int i = 3; i >= 0; i--) { + long segment = remaining % 0x100; + remaining = remaining / 0x100; + segments[i] = String.valueOf(segment); + } + return StringUtils.join(segments, IPSegmentsSeparator); + } + + public static boolean validateAsIP(String ipAddress, boolean ipInDotNotation) { + if (!ipInDotNotation) { + return StringUtils.isNumeric(ipAddress); + } + + boolean ret = false; + + try { + byte[] bytes = InetAddress.getByName(ipAddress).getAddress(); + ret = true; + } + catch(UnknownHostException exception) { + LOG.error("RangerGeolocationData.validateAsIP() - Invalid address " + ipAddress); + } + + return ret; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + + toStringDump(sb); + + return sb.toString(); + } + + private StringBuilder toString(StringBuilder sb) { + sb.append("{") + .append("from=") + .append(RangerGeolocationData.unsignedIntToIPAddress(fromIPAddress)) + .append(", to=") + .append(RangerGeolocationData.unsignedIntToIPAddress(toIPAddress)) + .append(", location={"); + for (int i = 0; i < locationData.length; i++) { + sb.append(locationData[i]).append(", "); + } + sb.append("}"); + sb.append("}"); + return sb; + } + + private StringBuilder toStringDump(StringBuilder sb) { + sb.append(RangerGeolocationData.unsignedIntToIPAddress(fromIPAddress)) + .append(",") + .append(RangerGeolocationData.unsignedIntToIPAddress(toIPAddress)) + .append(","); + for (int i = 0; i < locationData.length; i++) { + sb.append(locationData[i]).append(", "); + } + return sb; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/31a65f48/agents-common/src/main/java/org/apache/ranger/plugin/geo/RangerGeolocationDatabase.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/geo/RangerGeolocationDatabase.java b/agents-common/src/main/java/org/apache/ranger/plugin/geo/RangerGeolocationDatabase.java new file mode 100644 index 0000000..49cd4e9 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/geo/RangerGeolocationDatabase.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.geo; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class RangerGeolocationDatabase { + private static final Log LOG = LogFactory.getLog(RangerGeolocationDatabase.class); + + private BinarySearchTree<RangerGeolocationData, Long> data = new BinarySearchTree<RangerGeolocationData, Long>(); + + private GeolocationMetadata metadata = new GeolocationMetadata(); + + public String getValue(final RangerGeolocationData geolocationData, final String attributeName) { + String value = null; + int index = -1; + + if (geolocationData != null && StringUtils.isNotBlank(attributeName)) { + if ((index = getMetadata().getDataItemNameIndex(attributeName)) != -1) { + String[] attrValues = geolocationData.getLocationData(); + if (index < attrValues.length) { + value = attrValues[index]; + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerGeolocationDatabase.getValue() - No value specified attribute-name:" + attributeName); + } + } + } else { + LOG.error("RangerGeolocationDatabase.getValue() - RangerGeolocationDatabase not initialized or Invalid attribute-name:" + attributeName); + } + } + + return value; + } + + public RangerGeolocationData find(final String ipAddressStr) { + RangerGeolocationData ret = null; + + if (StringUtils.isNotBlank(ipAddressStr)) { + if (RangerGeolocationData.validateAsIP(ipAddressStr, true)) { + ret = data.find(RangerGeolocationData.ipAddressToLong(ipAddressStr)); + } + } + return ret; + } + + public void optimize() { + long start = 0L, end = 0L; + + start = System.currentTimeMillis(); + data.rebalance(); + end = System.currentTimeMillis(); + + if (LOG.isDebugEnabled()) { + LOG.debug("RangerGeolocationDatabase.optimize() - Time taken for optimizing database = " + (end - start) + " milliseconds"); + } + } + + public void setData(final BinarySearchTree<RangerGeolocationData, Long> dataArg) { data = dataArg != null ? dataArg : new BinarySearchTree<RangerGeolocationData, Long>();} + + public void setMetadata(final GeolocationMetadata metadataArg) { metadata = metadataArg != null ? metadataArg : new GeolocationMetadata();} + + public GeolocationMetadata getMetadata() { return metadata; } + + public BinarySearchTree<RangerGeolocationData, Long> getData() { return data; } + + public void dump(ValuePrinter<RangerGeolocationData> processor) { + + BinarySearchTree<RangerGeolocationData, Long> geoDatabase = getData(); + GeolocationMetadata metadata = getMetadata(); + processor.build(); + + processor.print("#================== Geolocation metadata =================="); + processor.print(metadata.toString()); + + processor.print("#================== Dump of geoDatabase - START =================="); + geoDatabase.preOrderTraverseTree(processor); + processor.print("#================== Dump of geoDatabase - END =================="); + + processor.close(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/31a65f48/agents-common/src/main/java/org/apache/ranger/plugin/geo/ValuePrinter.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/geo/ValuePrinter.java b/agents-common/src/main/java/org/apache/ranger/plugin/geo/ValuePrinter.java new file mode 100644 index 0000000..c353a6f --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/geo/ValuePrinter.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.geo; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.io.*; + +class ValuePrinter<T> implements ValueProcessor<T> { + private static final Log LOG = LogFactory.getLog(RangerGeolocationDatabase.class); + + private Writer writer; + private String fileName; + private boolean pretty; + + ValuePrinter(String fileName, boolean pretty) { + this.fileName = fileName; + this.pretty = pretty; + } + + public T process(T value) { + + if (value != null) { + if (writer == null) { + LOG.error("ValuePrinter.process() -" + value.toString()); + } else { + try { + writer.write(value.toString()); + writer.write('\n'); + } catch (IOException exception) { + LOG.error("ValuePrinter.process() - Cannot write '" + value.toString() + "' to " + fileName); + + } + } + } + return value; + } + + public void print(String str) { + if (writer == null) { + LOG.error("ValuePrinter.print() -" + str); + } else { + try { + writer.write(str); + writer.write('\n'); + } catch (IOException exception) { + LOG.error("ValuePrinter.print() - Cannot write '" + str + "' to " + fileName ); + } + } + } + + void build() { + try { + if (StringUtils.isNotBlank(fileName)) { + writer = new BufferedWriter(new OutputStreamWriter( + new FileOutputStream(fileName))); + } + } catch(IOException exception) { + LOG.error("ValuePrinter.build() - Cannot open " + fileName + " for writing"); + } + } + + void close() { + try { + if (writer != null) { + writer.close(); + } + } catch (IOException exception) { + LOG.error("ValuePrinter.close() - Cannot close " + fileName); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/31a65f48/agents-common/src/main/java/org/apache/ranger/plugin/geo/ValueProcessor.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/geo/ValueProcessor.java b/agents-common/src/main/java/org/apache/ranger/plugin/geo/ValueProcessor.java new file mode 100644 index 0000000..adae279 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/geo/ValueProcessor.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.geo; + +public interface ValueProcessor<T> { + T process(T value); +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/31a65f48/agents-common/src/main/java/org/apache/ranger/plugin/store/GeolocationStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/GeolocationStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/GeolocationStore.java new file mode 100644 index 0000000..340f971 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/GeolocationStore.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.store; + +import org.apache.ranger.plugin.geo.RangerGeolocationData; +import org.apache.ranger.plugin.geo.RangerGeolocationDatabase; + +import java.util.Map; + +public interface GeolocationStore { + void init(Map<String, String> context); + + RangerGeolocationData getGeoLocation(String ipAddress); + RangerGeolocationDatabase getGeoDatabase(); +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/31a65f48/agents-common/src/main/java/org/apache/ranger/plugin/store/file/GeolocationFileStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/GeolocationFileStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/GeolocationFileStore.java new file mode 100644 index 0000000..98ddd9c --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/GeolocationFileStore.java @@ -0,0 +1,256 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.store.file; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.geo.GeolocationMetadata; +import org.apache.ranger.plugin.store.GeolocationStore; +import org.apache.ranger.plugin.geo.RangerGeolocationDatabase; +import org.apache.ranger.plugin.geo.RangerGeolocationData; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.util.HashMap; +import java.util.Map; + +public class GeolocationFileStore implements GeolocationStore { + private static final Log LOG = LogFactory.getLog(GeolocationFileStore.class); + + public static final String GeoLineCommentIdentifier = "#"; + public static final Character GeoFieldsSeparator = ','; + + public static final String PROP_GEOLOCATION_FILE_LOCATION = "FilePath"; + public static final String PROP_GEOLOCATION_FILE_REINIT = "ForceRead"; + public static final String PROP_GEOLOCATION_IP_IN_DOT_FORMAT = "IPInDotFormat"; + + private static Map<String, RangerGeolocationDatabase> geolocationDBMap = new HashMap<>(); + + private RangerGeolocationDatabase geolocationDatabase; + + private boolean isMetalineProcessed = false; + private boolean useDotFormat = false; + + @Override + public void init(final Map<String, String> context) { + + String filePathToGeolocationFile = context.get(PROP_GEOLOCATION_FILE_LOCATION); + + if (StringUtils.isBlank(filePathToGeolocationFile)) { + filePathToGeolocationFile = "/etc/ranger/data/geo.txt"; + } + + String reinit = context.get(PROP_GEOLOCATION_FILE_REINIT); + boolean reinitialize = reinit == null || Boolean.parseBoolean(reinit); + + String ipInDotFormat = context.get(PROP_GEOLOCATION_IP_IN_DOT_FORMAT); + useDotFormat = ipInDotFormat == null || Boolean.parseBoolean(ipInDotFormat); + + + if (LOG.isDebugEnabled()) { + LOG.debug("GeolocationFileStore.init() - Geolocation file location=" + filePathToGeolocationFile); + LOG.debug("GeolocationFileStore.init() - Reinitialize flag =" + reinitialize); + LOG.debug("GeolocationFileStore.init() - UseDotFormat flag =" + useDotFormat); + } + + RangerGeolocationDatabase database = geolocationDBMap.get(filePathToGeolocationFile); + + if (database == null || reinitialize) { + RangerGeolocationDatabase newDatabase = build(filePathToGeolocationFile); + if (newDatabase != null) { + geolocationDBMap.put(filePathToGeolocationFile, newDatabase); + database = newDatabase; + } else { + LOG.error("GeolocationFileStore.init() - Could not build database. Using old database if present."); + } + } + geolocationDatabase = database; + + if (geolocationDatabase == null) { + LOG.error("GeolocationFileStore.init() - Cannot build Geolocation database from file " + filePathToGeolocationFile); + } + + } + + @Override + public RangerGeolocationDatabase getGeoDatabase() { + return geolocationDatabase; + } + + @Override + public final RangerGeolocationData getGeoLocation(final String ipAddress) { + RangerGeolocationData ret = null; + + RangerGeolocationDatabase database = geolocationDatabase; // init() may get called when getGeolocation is half-executed + + if (database != null) { + + long start = 0L, end = 0L; + + start = System.currentTimeMillis(); + ret = database.find(ipAddress); + end = System.currentTimeMillis(); + + if (LOG.isDebugEnabled()) { + if (ret == null) { + LOG.debug("GeolocationFileStore.getGeolocation() - " + ipAddress + " not found. Search time = " + (end - start) + " milliseconds"); + } else { + LOG.debug("GeolocationFileStore.getGeolocation() - " + ipAddress + " found. Search time = " + (end - start) + " milliseconds"); + + for (String attrName : database.getMetadata().getLocationDataItemNames()) { + LOG.debug("GeolocationFileStore.getGeolocation() - IPAddress[" + attrName + "]=" + database.getValue(ret, attrName) + ", "); + } + + } + } + } else { + LOG.error("GeolocationFileStore.getGeolocation() - GeoLocationDatabase is not initialized correctly."); + } + + return ret; + } + + private Reader getReader(String dataFileName) throws IOException { + Reader ret = null; + + File f = new File(dataFileName); + + if(f.exists() && f.canRead()) { + LOG.info("GeolocationFileStore: reading location data from file '" + dataFileName + "'"); + + ret = new FileReader(dataFileName); + } else { + InputStream inStr = this.getClass().getResourceAsStream(dataFileName); + + if(inStr != null) { + LOG.info("GeolocationFileStore: reading location data from resource '" + dataFileName + "'"); + + ret = new InputStreamReader(inStr); + } + } + + if(ret == null) { + throw new FileNotFoundException(dataFileName); + } + + return ret; + } + + RangerGeolocationDatabase build(String dataFileName) { + + RangerGeolocationDatabase database = null; + + BufferedReader bufferedReader = null; + long start = 0L, end = 0L; + + start = System.currentTimeMillis(); + + try { + bufferedReader = new BufferedReader(getReader(dataFileName)); + + database = new RangerGeolocationDatabase(); + + String line; + int lineNumber = 0; + isMetalineProcessed = false; + + while(( line = bufferedReader.readLine()) != null) { + lineNumber++; + if (!processLine(lineNumber, line, database)) { + LOG.error("RangerGeolocationDatabaseBuilder.build() - Invalid geo-specification - " + lineNumber + ":" + line); + database = null; + break; + } + } + + bufferedReader.close(); + bufferedReader = null; + } + catch(FileNotFoundException ex) { + LOG.error("RangerGeolocationDatabaseBuilder.build() - Unable to open file '" + dataFileName + "'"); + } + catch(IOException ex) { + LOG.error("RangerGeolocationDatabaseBuilder.build() - Error reading file '" + dataFileName + "', " + ex); + } + finally { + if (bufferedReader != null) { + try { + bufferedReader.close(); + } + catch (Exception exception) { + // Ignore + } + } + } + + end = System.currentTimeMillis(); + + if (LOG.isDebugEnabled()) { + LOG.debug("RangerGeolocationDatabaseBuilder.build() - Time taken for reading file = " + (end - start) + " milliseconds"); + } + + if (database != null) { + database.optimize(); + } + + return database; + } + + private boolean processLine(int lineNumber, String line, RangerGeolocationDatabase database) { + + boolean ret = true; + + line = line.trim(); + + if (!line.startsWith(GeoLineCommentIdentifier)) { + String fields[] = StringUtils.split(line, GeoFieldsSeparator); + if (fields != null) { + if (!isMetalineProcessed) { + GeolocationMetadata metadata = GeolocationMetadata.create(fields, lineNumber); + if (metadata != null) { + database.setMetadata(metadata); + isMetalineProcessed = true; + } else { + LOG.error("GeolocationFileStore.processLine() - Invalid metadata specification " + lineNumber + ":" + line); + ret = false; + } + } else { + RangerGeolocationData data = RangerGeolocationData.create(fields, lineNumber, useDotFormat); + if (data != null) { + database.getData().insert(data); + } else { + LOG.error("GeolocationFileStore.processLine() - Invalid data specification " + lineNumber + ":" + line); + } + } + } else { + LOG.error("GeolocationFileStore.processLine() - Invalid line, skipping.." + lineNumber + ":" + line); + } + } + return ret; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/31a65f48/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java ---------------------------------------------------------------------- diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java index fd39193..a7ce78d 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java @@ -41,6 +41,9 @@ import org.junit.Test; import java.io.*; import java.lang.reflect.Type; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.StandardCopyOption; import java.util.List; import java.util.Map; import java.util.Properties; @@ -98,6 +101,7 @@ public class TestPolicyEngine { System.out.println("provider=" + provider.toString()); +/* // For setting up TestTagProvider Path filePath = new Path("file:///tmp/ranger-admin-test-site.xml"); @@ -106,23 +110,25 @@ public class TestPolicyEngine { FileSystem fs = filePath.getFileSystem(config); FSDataOutputStream outStream = fs.create(filePath, true); - OutputStreamWriter writer = null; - - writer = new OutputStreamWriter(outStream); + OutputStreamWriter writer = new OutputStreamWriter(outStream); writer.write("<configuration>\n" + " <property>\n" + " <name>ranger.plugin.tag.policy.rest.url</name>\n" + " <value>http://node-1.example.com:6080</value>\n" + " </property>\n" + + " <property>\n" + + " <name>ranger.externalurl</name>\n" + + " <value>http://node-1.example.com:6080</value>\n" + + " </property>\n" + "</configuration>\n"); writer.close(); RangerConfiguration rangerConfig = RangerConfiguration.getInstance(); rangerConfig.addResource(filePath); - +*/ } @AfterClass http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/31a65f48/agents-common/src/test/resources/geo/geo.txt ---------------------------------------------------------------------- diff --git a/agents-common/src/test/resources/geo/geo.txt b/agents-common/src/test/resources/geo/geo.txt new file mode 100644 index 0000000..6e38db3 --- /dev/null +++ b/agents-common/src/test/resources/geo/geo.txt @@ -0,0 +1,29 @@ +# This is a sample geolocation data format used by Ranger +# If a line contains '#' as a first-nonblank character then it is considered a comment line +# First non-comment line in the file must be metadata line; metadata line contains '!' as first character +# Format of metadata and data lines is strictly Comma-Separated-Values. Spaces are not allowed to surround commas. +# Only IP-4 address values in dot-notation are supported. +# +FROM_IP,TO_IP,COUNTRY_CODE,COUNTRY_NAME,STATE,CITY,ZIP,LAT,LONG +10.0.0.255,10.0.3.0,US,United States,CA +20.0.100.100,20.0.100.109,US,United States,MT +20.0.100.80,20.0.100.89,CA,Canada +20.0.100.110,20.0.100.119,BR,Brazil +20.0.100.70,20.0.100.73,IN,India +20.0.100.120,20.0.100.129,NG,Nigeria +20.0.100.90,20.0.100.94,AUS,Austalia +20.0.100.74,20.0.100.79,AN,Angola +20.0.100.95,20.0.100.99,UK,United Kingdom +192.128.100.1,192.128.100.75,FR,France +1.0.0.0,1.0.0.255,AU,Australia,Queensland,Brisbane +1.0.1.0,1.0.3.255,CN,China,Fujian,Fuzhou, +1.0.4.0,1.0.4.255,AU,Australia,Victoria,Melbourne, +1.0.5.0,1.0.7.255,AU,Australia,-,-, +1.0.8.0,1.0.15.255,CN,China,Guangdong,Guangzhou, +255.255.255.0,255.255.255.255,CN,China,Guangdong,Guangzhou, +1.0.16.0,1.0.31.255,JP,Japan,Tokyo,Tokyo, +1.0.32.0,1.0.63.255,CN,China,Guangdong,Guangzhou, +1.0.64.0,1.0.79.255,JP,Japan,Hiroshima,Hiroshima, +1.0.80.0,1.0.83.255,JP,Japan,Tokyo,Tokyo, +128.101.101.100,128.101.101.255,US,United States,Minnesota,Minneapolis +#16797696,16798719,JP,Japan,Tokyo,Tokyo http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/31a65f48/agents-common/src/test/resources/policyengine/test_policyengine_hdfs.json ---------------------------------------------------------------------- diff --git a/agents-common/src/test/resources/policyengine/test_policyengine_hdfs.json b/agents-common/src/test/resources/policyengine/test_policyengine_hdfs.json index eed71be..53735f6 100644 --- a/agents-common/src/test/resources/policyengine/test_policyengine_hdfs.json +++ b/agents-common/src/test/resources/policyengine/test_policyengine_hdfs.json @@ -11,6 +11,28 @@ {"name":"read","label":"Read"}, {"name":"write","label":"Write"}, {"name":"execute","label":"Execute"} + ], + "contextEnrichers": + [ + { + "itemId":1, + "name" : "GeolocationEnricher", + "enricher" : "org.apache.ranger.plugin.contextenricher.RangerFileBasedGeolocationProvider", + "enricherOptions" : { + "geolocation.source.loader.options": "{'FilePath':'/geo/geo.txt', 'ForceRead':'false', 'IPInDotFormat':'true' }" + ,"geolocation.meta.prefix": "TEST_" + } + } + ], + "policyConditions": [ + { + "itemId":1, + "name":"ScriptConditionEvaluator", + "evaluator": "org.apache.ranger.plugin.conditionevaluator.RangerScriptConditionEvaluator", + "evaluatorOptions" : {"engineName":"JavaScript"}, + "label":"Script", + "description": "Script to execute" + } ] }, @@ -32,24 +54,48 @@ {"id":3,"name":"allow-read-to-finance under /finance/restricted","isEnabled":true,"isAuditEnabled":true, "resources":{"path":{"values":["/finance/restricted"],"isRecursive":true}}, "policyItems":[ - {"accesses":[{"type":"read","isAllowed":true}],"users":[],"groups":["finance"],"delegateAdmin":false} + {"accesses":[{"type":"read","isAllowed":true}],"users":[],"groups":["finance"],"delegateAdmin":false, + "conditions":[{ + "type":"ScriptConditionEvaluator", + "values":["var country_code = ctx.getGeolocation('LOCATION_TEST_COUNTRY_CODE'); ctx.result = !!country_code;"] + }]} ] } ], "tests":[ - {"name":"ALLOW 'read /finance/restricted/sales.db' for g=finance", + {"name":"ALLOW 'read /finance/restricted/sales.db' for g=finance; valid clientIPAddress", "request":{ "resource":{"elements":{"path":"/finance/restricted/sales.db"}}, - "accessType":"read","user":"user1","userGroups":["finance"],"requestData":"read /finance/restricted/sales.db" + "accessType":"read","user":"user1","userGroups":["finance"],"requestData":"read /finance/restricted/sales.db", + "clientIPAddress":"255.255.255.255" }, "result":{"isAudited":true,"isAllowed":true,"policyId":3} } , + {"name":"DENY 'read /finance/restricted/sales.db' for g=finance; invalid clientIPAddress", + "request":{ + "resource":{"elements":{"path":"/finance/restricted/sales.db"}}, + "accessType":"read","user":"user1","userGroups":["finance"],"requestData":"read /finance/restricted/sales.db", + "clientIPAddress":"128.101.101.99" + }, + "result":{"isAudited":true,"isAllowed":false,"policyId":-1} + } + , + {"name":"ALLOW 'read /finance/restricted/sales.db' for g=finance; no clientIPAddress", + "request":{ + "resource":{"elements":{"path":"/finance/restricted/sales.db"}}, + "accessType":"read","user":"user1","userGroups":["finance"],"requestData":"read /finance/restricted/sales.db" + }, + "result":{"isAudited":true,"isAllowed":false,"policyId":-1} + } + , {"name":"ALLOW 'read /finance/restricted/hr/payroll.db' for g=finance", "request":{ "resource":{"elements":{"path":"/finance/restricted/hr/payroll.db"}}, - "accessType":"read","user":"user1","userGroups":["finance"],"requestData":"read /finance/restricted/hr/payroll.db" + "accessType":"read","user":"user1","userGroups":["finance"],"requestData":"read /finance/restricted/hr/payroll.db", + "clientIPAddress":"128.101.101.101" + }, "result":{"isAudited":true,"isAllowed":true,"policyId":3} } @@ -57,7 +103,8 @@ {"name":"DENY 'read /operations/visitors.db' for g=finance", "request":{ "resource":{"elements":{"path":"/operations/visitors.db"}}, - "accessType":"read","user":"user1","userGroups":["finance"],"requestData":"read /operations/visitors.db" + "accessType":"read","user":"user1","userGroups":["finance"],"requestData":"read /operations/visitors.db", + "clientIPAddress":"128.101.101.99" }, "result":{"isAudited":false,"isAllowed":false,"policyId":-1} } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/31a65f48/knox-agent/src/main/java/org/apache/ranger/admin/client/RangerAdminJersey2RESTClient.java ---------------------------------------------------------------------- diff --git a/knox-agent/src/main/java/org/apache/ranger/admin/client/RangerAdminJersey2RESTClient.java b/knox-agent/src/main/java/org/apache/ranger/admin/client/RangerAdminJersey2RESTClient.java index 1ebb52f..b3ddcdc 100644 --- a/knox-agent/src/main/java/org/apache/ranger/admin/client/RangerAdminJersey2RESTClient.java +++ b/knox-agent/src/main/java/org/apache/ranger/admin/client/RangerAdminJersey2RESTClient.java @@ -237,7 +237,6 @@ public class RangerAdminJersey2RESTClient implements RangerAdminClient { public RangerTaggedResource updateTagsForResource(RangerTaggedResourceKey key, List<RangerTaggedResource.RangerResourceTag> tagsToAdd, List<RangerTaggedResource.RangerResourceTag> tagsToDelete) throws Exception { throw new Exception("RangerAdminjersey2RESTClient.updateTagsForResource() -- *** NOT IMPLEMENTED *** "); - } // package level methods left so (and not private only for testability!) Not intended for use outside this class!! http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/31a65f48/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 2be8b27..84ecf24 100644 --- a/pom.xml +++ b/pom.xml @@ -303,7 +303,7 @@ <artifactId>gson</artifactId> <version>${gson.version}</version> </dependency> - </dependencies> + </dependencies> </dependencyManagement> <build> <pluginManagement>
