Support for Solr as Audit Destination. Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/40aa090d Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/40aa090d Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/40aa090d
Branch: refs/heads/master Commit: 40aa090d7b23a524b9900f5b722d02b48c06b947 Parents: 8e6acd5 Author: Don Bosco Durai <[email protected]> Authored: Tue Mar 17 14:17:30 2015 -0700 Committer: Don Bosco Durai <[email protected]> Committed: Tue Mar 17 14:17:30 2015 -0700 ---------------------------------------------------------------------- agents-audit/pom.xml | 28 +- .../audit/provider/AuditProviderFactory.java | 23 +- .../audit/provider/BaseAuditProvider.java | 5 +- .../provider/kafka/KafkaAuditProvider.java | 2 +- .../audit/provider/solr/SolrAuditProvider.java | 275 ++++ hbase-agent/conf/ranger-hbase-audit-changes.cfg | 12 + hbase-agent/conf/ranger-hbase-audit.xml | 48 + hbase-agent/scripts/install.properties | 6 + hdfs-agent/conf/ranger-hdfs-audit-changes.cfg | 11 + hdfs-agent/conf/ranger-hdfs-audit.xml | 48 + hdfs-agent/scripts/install.properties | 7 + hive-agent/conf/ranger-hive-audit-changes.cfg | 12 + hive-agent/conf/ranger-hive-audit.xml | 48 + hive-agent/scripts/install.properties | 6 + knox-agent/conf/ranger-knox-audit-changes.cfg | 12 + knox-agent/conf/ranger-knox-audit.xml | 48 + knox-agent/scripts/install.properties | 6 + plugin-yarn/conf/ranger-yarn-audit-changes.cfg | 12 + plugin-yarn/conf/ranger-yarn-audit.xml | 48 + plugin-yarn/scripts/install.properties | 6 + pom.xml | 9 + ranger_solrj/.gitignore | 1 + ranger_solrj/pom.xml | 55 + .../solr/client/solrj/ResponseParser.java | 53 + .../apache/solr/client/solrj/SolrClient.java | 416 ++++++ .../org/apache/solr/client/solrj/SolrQuery.java | 1109 ++++++++++++++++ .../apache/solr/client/solrj/SolrRequest.java | 137 ++ .../apache/solr/client/solrj/SolrResponse.java | 65 + .../apache/solr/client/solrj/SolrServer.java | 25 + .../solr/client/solrj/SolrServerException.java | 54 + .../client/solrj/StreamingResponseCallback.java | 37 + .../client/solrj/beans/BindingException.java | 29 + .../solrj/beans/DocumentObjectBinder.java | 470 +++++++ .../apache/solr/client/solrj/beans/Field.java | 38 + .../solr/client/solrj/beans/package-info.java | 23 + .../client/solrj/impl/BinaryRequestWriter.java | 120 ++ .../client/solrj/impl/BinaryResponseParser.java | 64 + .../solr/client/solrj/impl/CloudSolrClient.java | 1232 ++++++++++++++++++ .../solr/client/solrj/impl/CloudSolrServer.java | 61 + .../solrj/impl/ConcurrentUpdateSolrClient.java | 492 +++++++ .../solrj/impl/ConcurrentUpdateSolrServer.java | 46 + .../client/solrj/impl/HttpClientConfigurer.java | 97 ++ .../solr/client/solrj/impl/HttpClientUtil.java | 365 ++++++ .../solr/client/solrj/impl/HttpSolrClient.java | 821 ++++++++++++ .../solr/client/solrj/impl/HttpSolrServer.java | 41 + .../client/solrj/impl/LBHttpSolrClient.java | 730 +++++++++++ .../client/solrj/impl/LBHttpSolrServer.java | 43 + .../client/solrj/impl/NoOpResponseParser.java | 83 ++ .../impl/StreamingBinaryResponseParser.java | 91 ++ .../client/solrj/impl/XMLResponseParser.java | 465 +++++++ .../solr/client/solrj/impl/package-info.java | 24 + .../apache/solr/client/solrj/package-info.java | 23 + .../solrj/request/AbstractUpdateRequest.java | 144 ++ .../solrj/request/CollectionAdminRequest.java | 860 ++++++++++++ .../request/ContentStreamUpdateRequest.java | 78 ++ .../client/solrj/request/CoreAdminRequest.java | 593 +++++++++ .../client/solrj/request/DirectXmlRequest.java | 66 + .../solrj/request/DocumentAnalysisRequest.java | 199 +++ .../solrj/request/FieldAnalysisRequest.java | 270 ++++ .../client/solrj/request/IsUpdateRequest.java | 26 + .../request/JavaBinUpdateRequestCodec.java | 251 ++++ .../solr/client/solrj/request/LukeRequest.java | 120 ++ .../solr/client/solrj/request/QueryRequest.java | 89 ++ .../client/solrj/request/RequestWriter.java | 146 +++ .../solr/client/solrj/request/SolrPing.java | 111 ++ .../client/solrj/request/UpdateRequest.java | 463 +++++++ .../solr/client/solrj/request/package-info.java | 23 + .../solrj/response/AnalysisResponseBase.java | 252 ++++ .../solrj/response/CollectionAdminResponse.java | 79 ++ .../solrj/response/CoreAdminResponse.java | 58 + .../response/DocumentAnalysisResponse.java | 258 ++++ .../solr/client/solrj/response/FacetField.java | 176 +++ .../solrj/response/FieldAnalysisResponse.java | 204 +++ .../client/solrj/response/FieldStatsInfo.java | 191 +++ .../solr/client/solrj/response/Group.java | 69 + .../client/solrj/response/GroupCommand.java | 125 ++ .../client/solrj/response/GroupResponse.java | 56 + .../client/solrj/response/IntervalFacet.java | 85 ++ .../client/solrj/response/LukeResponse.java | 270 ++++ .../solr/client/solrj/response/PivotField.java | 97 ++ .../client/solrj/response/QueryResponse.java | 586 +++++++++ .../solr/client/solrj/response/RangeFacet.java | 126 ++ .../client/solrj/response/SolrPingResponse.java | 28 + .../client/solrj/response/SolrResponseBase.java | 91 ++ .../solrj/response/SpellCheckResponse.java | 273 ++++ .../client/solrj/response/TermsResponse.java | 89 ++ .../client/solrj/response/UpdateResponse.java | 30 + .../client/solrj/response/package-info.java | 24 + .../solr/client/solrj/util/ClientUtils.java | 251 ++++ .../solr/client/solrj/util/package-info.java | 23 + .../org/apache/solr/common/EnumFieldValue.java | 116 ++ .../org/apache/solr/common/SolrDocument.java | 396 ++++++ .../apache/solr/common/SolrDocumentList.java | 68 + .../org/apache/solr/common/SolrException.java | 208 +++ .../apache/solr/common/SolrInputDocument.java | 301 +++++ .../org/apache/solr/common/SolrInputField.java | 232 ++++ .../org/apache/solr/common/StringUtils.java | 26 + .../org/apache/solr/common/cloud/Aliases.java | 63 + .../solr/common/cloud/BeforeReconnect.java | 22 + .../solr/common/cloud/ClosableThread.java | 27 + .../apache/solr/common/cloud/ClusterState.java | 397 ++++++ .../solr/common/cloud/ClusterStateUtil.java | 230 ++++ .../solr/common/cloud/CompositeIdRouter.java | 327 +++++ .../solr/common/cloud/ConnectionManager.java | 237 ++++ .../common/cloud/DefaultConnectionStrategy.java | 75 ++ .../solr/common/cloud/DefaultZkACLProvider.java | 45 + .../cloud/DefaultZkCredentialsProvider.java | 41 + .../apache/solr/common/cloud/DocCollection.java | 201 +++ .../org/apache/solr/common/cloud/DocRouter.java | 227 ++++ .../solr/common/cloud/HashBasedRouter.java | 81 ++ .../solr/common/cloud/ImplicitDocRouter.java | 104 ++ .../apache/solr/common/cloud/OnReconnect.java | 22 + .../apache/solr/common/cloud/PlainIdRouter.java | 23 + .../org/apache/solr/common/cloud/Replica.java | 48 + .../apache/solr/common/cloud/RoutingRule.java | 71 + .../solr/common/cloud/SaslZkACLProvider.java | 49 + .../org/apache/solr/common/cloud/Slice.java | 196 +++ .../apache/solr/common/cloud/SolrZkClient.java | 736 +++++++++++ .../apache/solr/common/cloud/SolrZooKeeper.java | 103 ++ ...ParamsAllAndReadonlyDigestZkACLProvider.java | 89 ++ ...tCredentialsDigestZkCredentialsProvider.java | 60 + .../apache/solr/common/cloud/ZkACLProvider.java | 28 + .../cloud/ZkClientConnectionStrategy.java | 113 ++ .../apache/solr/common/cloud/ZkCmdExecutor.java | 111 ++ .../solr/common/cloud/ZkConfigManager.java | 145 +++ .../solr/common/cloud/ZkCoreNodeProps.java | 74 ++ .../common/cloud/ZkCredentialsProvider.java | 45 + .../apache/solr/common/cloud/ZkNodeProps.java | 154 +++ .../apache/solr/common/cloud/ZkOperation.java | 37 + .../apache/solr/common/cloud/ZkStateReader.java | 925 +++++++++++++ .../solr/common/cloud/ZooKeeperException.java | 33 + .../apache/solr/common/cloud/package-info.java | 23 + .../org/apache/solr/common/luke/FieldFlag.java | 70 + .../apache/solr/common/luke/package-info.java | 23 + .../org/apache/solr/common/package-info.java | 23 + .../solr/common/params/AnalysisParams.java | 60 + .../solr/common/params/AppendedSolrParams.java | 55 + .../solr/common/params/CollectionParams.java | 74 ++ .../apache/solr/common/params/CommonParams.java | 228 ++++ .../solr/common/params/CoreAdminParams.java | 151 +++ .../solr/common/params/CursorMarkParams.java | 48 + .../solr/common/params/DefaultSolrParams.java | 68 + .../apache/solr/common/params/DisMaxParams.java | 78 ++ .../apache/solr/common/params/EventParams.java | 29 + .../apache/solr/common/params/ExpandParams.java | 32 + .../apache/solr/common/params/FacetParams.java | 405 ++++++ .../apache/solr/common/params/GroupParams.java | 71 + .../solr/common/params/HighlightParams.java | 82 ++ .../solr/common/params/MapSolrParams.java | 88 ++ .../common/params/ModifiableSolrParams.java | 210 +++ .../solr/common/params/MoreLikeThisParams.java | 74 ++ .../solr/common/params/MultiMapSolrParams.java | 92 ++ .../common/params/QueryElevationParams.java | 53 + .../solr/common/params/RequiredSolrParams.java | 155 +++ .../apache/solr/common/params/ShardParams.java | 56 + .../apache/solr/common/params/SimpleParams.java | 50 + .../apache/solr/common/params/SolrParams.java | 363 ++++++ .../solr/common/params/SpatialParams.java | 41 + .../solr/common/params/SpellingParams.java | 174 +++ .../apache/solr/common/params/StatsParams.java | 28 + .../solr/common/params/TermVectorParams.java | 66 + .../apache/solr/common/params/TermsParams.java | 120 ++ .../apache/solr/common/params/UpdateParams.java | 72 + .../apache/solr/common/params/package-info.java | 22 + .../org/apache/solr/common/util/Base64.java | 153 +++ .../org/apache/solr/common/util/ByteUtils.java | 126 ++ .../apache/solr/common/util/ContentStream.java | 81 ++ .../solr/common/util/ContentStreamBase.java | 260 ++++ .../solr/common/util/DataInputInputStream.java | 27 + .../org/apache/solr/common/util/DateUtil.java | 260 ++++ .../apache/solr/common/util/ExecutorUtil.java | 64 + .../solr/common/util/FastInputStream.java | 253 ++++ .../solr/common/util/FastOutputStream.java | 233 ++++ .../java/org/apache/solr/common/util/Hash.java | 422 ++++++ .../org/apache/solr/common/util/IOUtils.java | 37 + .../apache/solr/common/util/IteratorChain.java | 87 ++ .../apache/solr/common/util/JavaBinCodec.java | 820 ++++++++++++ .../solr/common/util/JsonRecordReader.java | 586 +++++++++ .../org/apache/solr/common/util/NamedList.java | 708 ++++++++++ .../solr/common/util/ObjectReleaseTracker.java | 62 + .../org/apache/solr/common/util/RetryUtil.java | 43 + .../solr/common/util/SimpleOrderedMap.java | 67 + .../common/util/SolrjNamedThreadFactory.java | 50 + .../org/apache/solr/common/util/StrUtils.java | 309 +++++ .../org/apache/solr/common/util/URLUtil.java | 50 + .../java/org/apache/solr/common/util/XML.java | 207 +++ .../apache/solr/common/util/XMLErrorLogger.java | 84 ++ .../apache/solr/common/util/package-info.java | 23 + ranger_solrj/src/main/java/overview.html | 21 + security-admin/pom.xml | 5 + security-admin/scripts/install.properties | 10 + security-admin/scripts/setup.sh | 48 +- .../java/org/apache/ranger/biz/AssetMgr.java | 11 +- .../org/apache/ranger/biz/RangerBizUtil.java | 434 +++--- .../java/org/apache/ranger/biz/XAuditMgr.java | 41 +- .../apache/ranger/db/RangerDaoManagerBase.java | 3 + .../ranger/solr/SolrAccessAuditsService.java | 253 ++++ .../java/org/apache/ranger/solr/SolrMgr.java | 99 ++ .../java/org/apache/ranger/solr/SolrUtil.java | 327 +++++ src/main/assembly/hbase-agent.xml | 6 + src/main/assembly/hdfs-agent.xml | 6 + src/main/assembly/hive-agent.xml | 6 + src/main/assembly/knox-agent.xml | 6 + src/main/assembly/storm-agent.xml | 6 + storm-agent/conf/ranger-storm-audit-changes.cfg | 13 + storm-agent/conf/ranger-storm-audit.xml | 48 + storm-agent/scripts/install.properties | 6 + 207 files changed, 30674 insertions(+), 219 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/agents-audit/pom.xml ---------------------------------------------------------------------- diff --git a/agents-audit/pom.xml b/agents-audit/pom.xml index e54ec36..6715575 100644 --- a/agents-audit/pom.xml +++ b/agents-audit/pom.xml @@ -63,9 +63,29 @@ <version>${log4j.version}</version> </dependency> <dependency> - <groupId>org.apache.kafka</groupId> - <artifactId>kafka_2.10</artifactId> - <version>0.8.2.0</version> -</dependency> + <groupId>org.apache.kafka</groupId> + <artifactId>kafka_2.10</artifactId> + <version>${kafka.version}</version> + </dependency> + <dependency> + <groupId>org.apache.ranger</groupId> + <artifactId>ranger_solrj</artifactId> + <version>${ranger.solrj.version}</version> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + <version>${httpcomponent.httpclient.version}</version> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpmime</artifactId> + <version>${httpcomponent.httpmime.version}</version> + </dependency> + <dependency> + <groupId>org.noggit</groupId> + <artifactId>noggit</artifactId> + <version>${noggit.version}</version> + </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java index 9fbe29f..bb8fa6d 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java @@ -26,6 +26,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.ranger.audit.provider.hdfs.HdfsAuditProvider; import org.apache.ranger.audit.provider.kafka.KafkaAuditProvider; +import org.apache.ranger.audit.provider.solr.SolrAuditProvider; /* @@ -43,6 +44,7 @@ public class AuditProviderFactory { private static final String AUDIT_HDFS_IS_ENABLED_PROP = "xasecure.audit.hdfs.is.enabled"; private static final String AUDIT_LOG4J_IS_ENABLED_PROP = "xasecure.audit.log4j.is.enabled" ; private static final String AUDIT_KAFKA_IS_ENABLED_PROP = "xasecure.audit.kafka.is.enabled"; + private static final String AUDIT_SOLR_IS_ENABLED_PROP = "xasecure.audit.solr.is.enabled"; private static final int AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT = 10 * 1024; private static final int AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT = 5 * 1000; @@ -99,8 +101,9 @@ public class AuditProviderFactory { boolean isAuditToHdfsEnabled = BaseAuditProvider.getBooleanProperty(props, AUDIT_HDFS_IS_ENABLED_PROP, false); boolean isAuditToLog4jEnabled = BaseAuditProvider.getBooleanProperty(props, AUDIT_LOG4J_IS_ENABLED_PROP, false); boolean isAuditToKafkaEnabled = BaseAuditProvider.getBooleanProperty(props, AUDIT_KAFKA_IS_ENABLED_PROP, false); + boolean isAuditToSolrEnabled = BaseAuditProvider.getBooleanProperty(props, AUDIT_SOLR_IS_ENABLED_PROP, false); - if(!isEnabled || !(isAuditToDbEnabled || isAuditToHdfsEnabled || isAuditToKafkaEnabled || isAuditToLog4jEnabled)) { + if(!isEnabled || !(isAuditToDbEnabled || isAuditToHdfsEnabled || isAuditToKafkaEnabled || isAuditToLog4jEnabled || isAuditToSolrEnabled)) { LOG.info("AuditProviderFactory: Audit not enabled.."); mProvider = getDefaultProvider(); @@ -111,6 +114,7 @@ public class AuditProviderFactory { List<AuditProvider> providers = new ArrayList<AuditProvider>(); if(isAuditToDbEnabled) { + LOG.info("DbAuditProvider is enabled"); DbAuditProvider dbProvider = new DbAuditProvider(); boolean isAuditToDbAsync = BaseAuditProvider.getBooleanProperty(props, DbAuditProvider.AUDIT_DB_IS_ASYNC_PROP, false); @@ -128,6 +132,8 @@ public class AuditProviderFactory { } if(isAuditToHdfsEnabled) { + LOG.info("HdfsAuditProvider is enabled"); + HdfsAuditProvider hdfsProvider = new HdfsAuditProvider(); boolean isAuditToHdfsAsync = BaseAuditProvider.getBooleanProperty(props, HdfsAuditProvider.AUDIT_HDFS_IS_ASYNC_PROP, false); @@ -156,7 +162,20 @@ public class AuditProviderFactory { providers.add(kafkaProvider); } } - + + if(isAuditToSolrEnabled) { + LOG.info("SolrAuditProvider is enabled"); + SolrAuditProvider solrProvider = new SolrAuditProvider(); + solrProvider.init(props); + + if( solrProvider.isAsync()) { + AsyncAuditProvider asyncProvider = new AsyncAuditProvider("MySolrAuditProvider", solrProvider.getMaxQueueSize(), solrProvider.getMaxFlushInterval(), solrProvider); + providers.add(asyncProvider); + } else { + providers.add(solrProvider); + } + } + if(isAuditToLog4jEnabled) { Log4jAuditProvider log4jProvider = new Log4jAuditProvider(); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditProvider.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditProvider.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditProvider.java index 14e6220..a068b8f 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditProvider.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditProvider.java @@ -40,7 +40,7 @@ public abstract class BaseAuditProvider implements AuditProvider { private int maxQueueSize = AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT; private int maxFlushInterval = AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT; - + protected Properties props = null; public BaseAuditProvider() { } @@ -48,7 +48,8 @@ public abstract class BaseAuditProvider implements AuditProvider { @Override public void init(Properties props) { LOG.info("BaseAuditProvider.init()"); - + this.props = props; + mLogFailureReportMinIntervalInMs = getIntProperty(props, AUDIT_LOG_FAILURE_REPORT_MIN_INTERVAL_PROP, 60 * 1000); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/agents-audit/src/main/java/org/apache/ranger/audit/provider/kafka/KafkaAuditProvider.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/kafka/KafkaAuditProvider.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/kafka/KafkaAuditProvider.java index 54e73ea..0ec8790 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/kafka/KafkaAuditProvider.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/kafka/KafkaAuditProvider.java @@ -117,7 +117,7 @@ public class KafkaAuditProvider extends BaseAuditProvider { } } catch (Throwable t) { LOG.error("Error sending message to Kafka topic. topic=" + topic - + ", message=" + message); + + ", message=" + message, t); } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java new file mode 100644 index 0000000..1b463e6 --- /dev/null +++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.provider.solr; + +import java.util.Date; +import java.util.Properties; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.audit.model.AuditEventBase; +import org.apache.ranger.audit.model.AuthzAuditEvent; +import org.apache.ranger.audit.provider.BaseAuditProvider; +import org.apache.ranger.audit.provider.MiscUtil; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.response.UpdateResponse; +import org.apache.solr.common.SolrInputDocument; + +public class SolrAuditProvider extends BaseAuditProvider { + private static final Log LOG = LogFactory.getLog(SolrAuditProvider.class); + + public static final String AUDIT_MAX_QUEUE_SIZE_PROP = "xasecure.audit.solr.async.max.queue.size"; + public static final String AUDIT_MAX_FLUSH_INTERVAL_PROP = "xasecure.audit.solr.async.max.flush.interval.ms"; + public static final String AUDIT_RETRY_WAIT_PROP = "xasecure.audit.solr.retry.ms"; + + static final Object lock = new Object(); + SolrClient solrClient = null; + Date lastConnectTime = null; + long lastFailTime = 0; + + int retryWaitTime = 30000; + + public SolrAuditProvider() { + } + + @Override + public void init(Properties props) { + LOG.info("init() called"); + super.init(props); + + setMaxQueueSize(BaseAuditProvider.getIntProperty(props, + AUDIT_MAX_QUEUE_SIZE_PROP, AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT)); + setMaxFlushInterval(BaseAuditProvider.getIntProperty(props, + AUDIT_MAX_QUEUE_SIZE_PROP, + AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT)); + retryWaitTime = BaseAuditProvider.getIntProperty(props, + AUDIT_RETRY_WAIT_PROP, retryWaitTime); + } + + void connect() { + if (solrClient == null) { + synchronized (lock) { + if (solrClient == null) { + String solrURL = BaseAuditProvider.getStringProperty(props, + "xasecure.audit.solr.solr_url"); + + if (lastConnectTime != null) { + // Let's wait for enough time before retrying + long diff = System.currentTimeMillis() + - lastConnectTime.getTime(); + if (diff < retryWaitTime) { + if (LOG.isDebugEnabled()) { + LOG.debug("Ignore connecting to solr url=" + + solrURL + ", lastConnect=" + diff + + "ms"); + } + return; + } + } + lastConnectTime = new Date(); + + if (solrURL == null || solrURL.isEmpty()) { + LOG.fatal("Solr URL for Audit is empty"); + return; + } + + try { + // TODO: Need to support SolrCloud also + solrClient = new HttpSolrClient(solrURL); + if (solrClient instanceof HttpSolrClient) { + HttpSolrClient httpSolrClient = (HttpSolrClient) solrClient; + httpSolrClient.setAllowCompression(true); + httpSolrClient.setConnectionTimeout(1000); + // solrClient.setSoTimeout(10000); + httpSolrClient.setMaxRetries(1); + } + } catch (Throwable t) { + LOG.fatal("Can't connect to Solr server. URL=" + + solrURL, t); + } + } + } + } + } + + /* + * (non-Javadoc) + * + * @see + * org.apache.ranger.audit.provider.AuditProvider#log(org.apache.ranger. + * audit.model.AuditEventBase) + */ + @Override + public void log(AuditEventBase event) { + if (!(event instanceof AuthzAuditEvent)) { + LOG.error(event.getClass().getName() + + " audit event class type is not supported"); + return; + } + AuthzAuditEvent authzEvent = (AuthzAuditEvent) event; + // TODO: This should be done at a higher level + + if (authzEvent.getAgentHostname() == null) { + authzEvent.setAgentHostname(MiscUtil.getHostname()); + } + + if (authzEvent.getLogType() == null) { + authzEvent.setLogType("RangerAudit"); + } + + if (authzEvent.getEventId() == null) { + authzEvent.setEventId(MiscUtil.generateUniqueId()); + } + + try { + if (solrClient == null) { + connect(); + if (solrClient == null) { + // Solr is still not initialized. So need to throw error + return; + } + } + + if (lastFailTime > 0) { + long diff = System.currentTimeMillis() - lastFailTime; + if (diff < retryWaitTime) { + if (LOG.isDebugEnabled()) { + LOG.debug("Ignore sending audit. lastConnect=" + diff + + " ms"); + } + return; + } + } + // Convert AuditEventBase to Solr document + SolrInputDocument document = toSolrDoc(authzEvent); + UpdateResponse response = solrClient.add(document); + if (response.getStatus() != 0) { + lastFailTime = System.currentTimeMillis(); + + // System.out.println("Response=" + response.toString() + // + ", status= " + response.getStatus() + ", event=" + // + event); + // throw new Exception("Aborting. event=" + event + + // ", response=" + // + response.toString()); + } else { + lastFailTime = 0; + } + + } catch (Throwable t) { + LOG.error("Error sending message to Solr", t); + } + + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#start() + */ + @Override + public void start() { + connect(); + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#stop() + */ + @Override + public void stop() { + // TODO Auto-generated method stub + + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#waitToComplete() + */ + @Override + public void waitToComplete() { + // TODO Auto-generated method stub + + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#isFlushPending() + */ + @Override + public boolean isFlushPending() { + // TODO Auto-generated method stub + return false; + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#getLastFlushTime() + */ + @Override + public long getLastFlushTime() { + // TODO Auto-generated method stub + return 0; + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#flush() + */ + @Override + public void flush() { + // TODO Auto-generated method stub + + } + + SolrInputDocument toSolrDoc(AuthzAuditEvent auditEvent) { + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("id", auditEvent.getEventId()); + doc.addField("access", auditEvent.getAccessType()); + doc.addField("enforcer", auditEvent.getAclEnforcer()); + doc.addField("agent", auditEvent.getAgentId()); + doc.addField("repo", auditEvent.getRepositoryName()); + doc.addField("sess", auditEvent.getSessionId()); + doc.addField("reqUser", auditEvent.getUser()); + doc.addField("reqData", auditEvent.getRequestData()); + doc.addField("resource", auditEvent.getResourcePath()); + doc.addField("cliIP", auditEvent.getClientIP()); + doc.addField("logType", auditEvent.getLogType()); + doc.addField("result", auditEvent.getAccessResult()); + doc.addField("policy", auditEvent.getPolicyId()); + doc.addField("repoType", auditEvent.getRepositoryType()); + doc.addField("resType", auditEvent.getResourceType()); + doc.addField("reason", auditEvent.getResultReason()); + doc.addField("action", auditEvent.getAction()); + doc.addField("evtTime", auditEvent.getEventTime()); + return doc; + } + + public boolean isAsync() { + return true; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hbase-agent/conf/ranger-hbase-audit-changes.cfg ---------------------------------------------------------------------- diff --git a/hbase-agent/conf/ranger-hbase-audit-changes.cfg b/hbase-agent/conf/ranger-hbase-audit-changes.cfg index e5c381a..221d20a 100644 --- a/hbase-agent/conf/ranger-hbase-audit-changes.cfg +++ b/hbase-agent/conf/ranger-hbase-audit-changes.cfg @@ -31,3 +31,15 @@ xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.local.archive.directory %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists + +#xasecure.audit.kafka.is.enabled %XAAUDIT.KAFKA.IS_ENABLED% mod create-if-not-exists +#xasecure.audit.kafka.is.async %XAAUDIT.KAFKA.IS_ASYNC% mod create-if-not-exists +#xasecure.audit.kafka.async.max.queue.size %XAAUDIT.KAFKA.MAX_QUEUE_SIZE% mod create-if-not-exists +#xasecure.audit.kafka.async.max.flush.interval.ms %XAAUDIT.KAFKA.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists +#xasecure.audit.kafka.broker_list %XAAUDIT.KAFKA.BROKER_LIST% mod create-if-not-exists +#xasecure.audit.kafka.topic_name %XAAUDIT.KAFKA.TOPIC_NAME% mod create-if-not-exists + +xasecure.audit.solr.is.enabled %XAAUDIT.SOLR.IS_ENABLED% mod create-if-not-exists +xasecure.audit.solr.async.max.queue.size %XAAUDIT.SOLR.MAX_QUEUE_SIZE% mod create-if-not-exists +xasecure.audit.solr.async.max.flush.interval.ms %XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists +xasecure.audit.solr.solr_url %XAAUDIT.SOLR.SOLR_URL% mod create-if-not-exists http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hbase-agent/conf/ranger-hbase-audit.xml ---------------------------------------------------------------------- diff --git a/hbase-agent/conf/ranger-hbase-audit.xml b/hbase-agent/conf/ranger-hbase-audit.xml index b39696b..e5bfb89 100644 --- a/hbase-agent/conf/ranger-hbase-audit.xml +++ b/hbase-agent/conf/ranger-hbase-audit.xml @@ -183,4 +183,52 @@ <name>xasecure.audit.log4j.async.max.flush.interval.ms</name> <value>30000</value> </property> + + <!-- Kafka audit provider configuration --> + <property> + <name>xasecure.audit.kafka.is.enabled</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.kafka.async.max.queue.size</name> + <value>1</value> + </property> + + <property> + <name>xasecure.audit.kafka.async.max.flush.interval.ms</name> + <value>1000</value> + </property> + + <property> + <name>xasecure.audit.kafka.broker_list</name> + <value>localhost:9092</value> + </property> + + <property> + <name>xasecure.audit.kafka.topic_name</name> + <value>ranger_audits</value> + </property> + + <!-- Ranger audit provider configuration --> + <property> + <name>xasecure.audit.ranger.is.enabled</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.ranger.async.max.queue.size</name> + <value>1</value> + </property> + + <property> + <name>xasecure.audit.ranger.async.max.flush.interval.ms</name> + <value>1000</value> + </property> + + <property> + <name>xasecure.audit.solr.solr_url</name> + <value>http://localhost:6083/solr/ranger_audits</value> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hbase-agent/scripts/install.properties ---------------------------------------------------------------------- diff --git a/hbase-agent/scripts/install.properties b/hbase-agent/scripts/install.properties index 5a81ad4..7ff29c9 100644 --- a/hbase-agent/scripts/install.properties +++ b/hbase-agent/scripts/install.properties @@ -89,6 +89,12 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 +#Solr Audit Provder +XAAUDIT.SOLR.IS_ENABLED=false +XAAUDIT.SOLR.MAX_QUEUE_SIZE=1 +XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000 +XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits + # # SSL Client Certificate Information # http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg ---------------------------------------------------------------------- diff --git a/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg b/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg index e5c381a..8d31016 100644 --- a/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg +++ b/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg @@ -31,3 +31,14 @@ xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.local.archive.directory %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists + +#xasecure.audit.kafka.is.enabled %XAAUDIT.KAFKA.IS_ENABLED% mod create-if-not-exists +#xasecure.audit.kafka.async.max.queue.size %XAAUDIT.KAFKA.MAX_QUEUE_SIZE% mod create-if-not-exists +#xasecure.audit.kafka.async.max.flush.interval.ms %XAAUDIT.KAFKA.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists +#xasecure.audit.kafka.broker_list %XAAUDIT.KAFKA.BROKER_LIST% mod create-if-not-exists +#xasecure.audit.kafka.topic_name %XAAUDIT.KAFKA.TOPIC_NAME% mod create-if-not-exists + +xasecure.audit.solr.is.enabled %XAAUDIT.SOLR.IS_ENABLED% mod create-if-not-exists +xasecure.audit.solr.async.max.queue.size %XAAUDIT.SOLR.MAX_QUEUE_SIZE% mod create-if-not-exists +xasecure.audit.solr.async.max.flush.interval.ms %XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists +xasecure.audit.solr.solr_url %XAAUDIT.SOLR.SOLR_URL% mod create-if-not-exists http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hdfs-agent/conf/ranger-hdfs-audit.xml ---------------------------------------------------------------------- diff --git a/hdfs-agent/conf/ranger-hdfs-audit.xml b/hdfs-agent/conf/ranger-hdfs-audit.xml index d26345d..09114ad 100644 --- a/hdfs-agent/conf/ranger-hdfs-audit.xml +++ b/hdfs-agent/conf/ranger-hdfs-audit.xml @@ -183,4 +183,52 @@ <name>xasecure.audit.log4j.async.max.flush.interval.ms</name> <value>30000</value> </property> + + <!-- Kafka audit provider configuration --> + <property> + <name>xasecure.audit.kafka.is.enabled</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.kafka.async.max.queue.size</name> + <value>1</value> + </property> + + <property> + <name>xasecure.audit.kafka.async.max.flush.interval.ms</name> + <value>1000</value> + </property> + + <property> + <name>xasecure.audit.kafka.broker_list</name> + <value>localhost:9092</value> + </property> + + <property> + <name>xasecure.audit.kafka.topic_name</name> + <value>ranger_audits</value> + </property> + + <!-- Ranger audit provider configuration --> + <property> + <name>xasecure.audit.ranger.is.enabled</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.ranger.async.max.queue.size</name> + <value>1</value> + </property> + + <property> + <name>xasecure.audit.ranger.async.max.flush.interval.ms</name> + <value>1000</value> + </property> + + <property> + <name>xasecure.audit.solr.solr_url</name> + <value>http://localhost:6083/solr/ranger_audits</value> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hdfs-agent/scripts/install.properties ---------------------------------------------------------------------- diff --git a/hdfs-agent/scripts/install.properties b/hdfs-agent/scripts/install.properties index 93790e3..2e1b61a 100644 --- a/hdfs-agent/scripts/install.properties +++ b/hdfs-agent/scripts/install.properties @@ -89,6 +89,13 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 +#Solr Audit Provder +XAAUDIT.SOLR.IS_ENABLED=false +XAAUDIT.SOLR.MAX_QUEUE_SIZE=1 +XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000 +XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits + + # # SSL Client Certificate Information # http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hive-agent/conf/ranger-hive-audit-changes.cfg ---------------------------------------------------------------------- diff --git a/hive-agent/conf/ranger-hive-audit-changes.cfg b/hive-agent/conf/ranger-hive-audit-changes.cfg index 9fa7608..2d6d414 100644 --- a/hive-agent/conf/ranger-hive-audit-changes.cfg +++ b/hive-agent/conf/ranger-hive-audit-changes.cfg @@ -31,3 +31,15 @@ xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.local.archive.directory %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists + +#xasecure.audit.kafka.is.enabled %XAAUDIT.KAFKA.IS_ENABLED% mod create-if-not-exists +#xasecure.audit.kafka.is.async %XAAUDIT.KAFKA.IS_ASYNC% mod create-if-not-exists +#xasecure.audit.kafka.async.max.queue.size %XAAUDIT.KAFKA.MAX_QUEUE_SIZE% mod create-if-not-exists +#xasecure.audit.kafka.async.max.flush.interval.ms %XAAUDIT.KAFKA.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists +#xasecure.audit.kafka.broker_list %XAAUDIT.KAFKA.BROKER_LIST% mod create-if-not-exists +#xasecure.audit.kafka.topic_name %XAAUDIT.KAFKA.TOPIC_NAME% mod create-if-not-exists + +xasecure.audit.solr.is.enabled %XAAUDIT.SOLR.IS_ENABLED% mod create-if-not-exists +xasecure.audit.solr.async.max.queue.size %XAAUDIT.SOLR.MAX_QUEUE_SIZE% mod create-if-not-exists +xasecure.audit.solr.async.max.flush.interval.ms %XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists +xasecure.audit.solr.solr_url %XAAUDIT.SOLR.SOLR_URL% mod create-if-not-exists http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hive-agent/conf/ranger-hive-audit.xml ---------------------------------------------------------------------- diff --git a/hive-agent/conf/ranger-hive-audit.xml b/hive-agent/conf/ranger-hive-audit.xml index d011b24..e753336 100644 --- a/hive-agent/conf/ranger-hive-audit.xml +++ b/hive-agent/conf/ranger-hive-audit.xml @@ -183,4 +183,52 @@ <name>xasecure.audit.log4j.async.max.flush.interval.ms</name> <value>30000</value> </property> + + <!-- Kafka audit provider configuration --> + <property> + <name>xasecure.audit.kafka.is.enabled</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.kafka.async.max.queue.size</name> + <value>1</value> + </property> + + <property> + <name>xasecure.audit.kafka.async.max.flush.interval.ms</name> + <value>1000</value> + </property> + + <property> + <name>xasecure.audit.kafka.broker_list</name> + <value>localhost:9092</value> + </property> + + <property> + <name>xasecure.audit.kafka.topic_name</name> + <value>ranger_audits</value> + </property> + + <!-- Ranger audit provider configuration --> + <property> + <name>xasecure.audit.ranger.is.enabled</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.ranger.async.max.queue.size</name> + <value>1</value> + </property> + + <property> + <name>xasecure.audit.ranger.async.max.flush.interval.ms</name> + <value>1000</value> + </property> + + <property> + <name>xasecure.audit.solr.solr_url</name> + <value>http://localhost:6083/solr/ranger_audits</value> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hive-agent/scripts/install.properties ---------------------------------------------------------------------- diff --git a/hive-agent/scripts/install.properties b/hive-agent/scripts/install.properties index bbd1849..75b1b5d 100644 --- a/hive-agent/scripts/install.properties +++ b/hive-agent/scripts/install.properties @@ -89,6 +89,12 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 +#Solr Audit Provder +XAAUDIT.SOLR.IS_ENABLED=false +XAAUDIT.SOLR.MAX_QUEUE_SIZE=1 +XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000 +XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits + # # SSL Client Certificate Information # http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/knox-agent/conf/ranger-knox-audit-changes.cfg ---------------------------------------------------------------------- diff --git a/knox-agent/conf/ranger-knox-audit-changes.cfg b/knox-agent/conf/ranger-knox-audit-changes.cfg index 7ae334e..f97d10f 100644 --- a/knox-agent/conf/ranger-knox-audit-changes.cfg +++ b/knox-agent/conf/ranger-knox-audit-changes.cfg @@ -31,3 +31,15 @@ xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.local.archive.directory %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists + +#xasecure.audit.kafka.is.enabled %XAAUDIT.KAFKA.IS_ENABLED% mod create-if-not-exists +#xasecure.audit.kafka.is.async %XAAUDIT.KAFKA.IS_ASYNC% mod create-if-not-exists +#xasecure.audit.kafka.async.max.queue.size %XAAUDIT.KAFKA.MAX_QUEUE_SIZE% mod create-if-not-exists +#xasecure.audit.kafka.async.max.flush.interval.ms %XAAUDIT.KAFKA.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists +#xasecure.audit.kafka.broker_list %XAAUDIT.KAFKA.BROKER_LIST% mod create-if-not-exists +#xasecure.audit.kafka.topic_name %XAAUDIT.KAFKA.TOPIC_NAME% mod create-if-not-exists + +xasecure.audit.solr.is.enabled %XAAUDIT.SOLR.IS_ENABLED% mod create-if-not-exists +xasecure.audit.solr.async.max.queue.size %XAAUDIT.SOLR.MAX_QUEUE_SIZE% mod create-if-not-exists +xasecure.audit.solr.async.max.flush.interval.ms %XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists +xasecure.audit.solr.solr_url %XAAUDIT.SOLR.SOLR_URL% mod create-if-not-exists http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/knox-agent/conf/ranger-knox-audit.xml ---------------------------------------------------------------------- diff --git a/knox-agent/conf/ranger-knox-audit.xml b/knox-agent/conf/ranger-knox-audit.xml index 0fdcefc..6f0adb9 100644 --- a/knox-agent/conf/ranger-knox-audit.xml +++ b/knox-agent/conf/ranger-knox-audit.xml @@ -183,4 +183,52 @@ <name>xasecure.audit.log4j.async.max.flush.interval.ms</name> <value>30000</value> </property> + + <!-- Kafka audit provider configuration --> + <property> + <name>xasecure.audit.kafka.is.enabled</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.kafka.async.max.queue.size</name> + <value>1</value> + </property> + + <property> + <name>xasecure.audit.kafka.async.max.flush.interval.ms</name> + <value>1000</value> + </property> + + <property> + <name>xasecure.audit.kafka.broker_list</name> + <value>localhost:9092</value> + </property> + + <property> + <name>xasecure.audit.kafka.topic_name</name> + <value>ranger_audits</value> + </property> + + <!-- Ranger audit provider configuration --> + <property> + <name>xasecure.audit.ranger.is.enabled</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.ranger.async.max.queue.size</name> + <value>1</value> + </property> + + <property> + <name>xasecure.audit.ranger.async.max.flush.interval.ms</name> + <value>1000</value> + </property> + + <property> + <name>xasecure.audit.solr.solr_url</name> + <value>http://localhost:6083/solr/ranger_audits</value> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/knox-agent/scripts/install.properties ---------------------------------------------------------------------- diff --git a/knox-agent/scripts/install.properties b/knox-agent/scripts/install.properties index d821c5d..ecd9813 100644 --- a/knox-agent/scripts/install.properties +++ b/knox-agent/scripts/install.properties @@ -92,6 +92,12 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 +#Solr Audit Provder +XAAUDIT.SOLR.IS_ENABLED=false +XAAUDIT.SOLR.MAX_QUEUE_SIZE=1 +XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000 +XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits + # # SSL Client Certificate Information # http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/plugin-yarn/conf/ranger-yarn-audit-changes.cfg ---------------------------------------------------------------------- diff --git a/plugin-yarn/conf/ranger-yarn-audit-changes.cfg b/plugin-yarn/conf/ranger-yarn-audit-changes.cfg index 4f2c5a2..e0dbea2 100644 --- a/plugin-yarn/conf/ranger-yarn-audit-changes.cfg +++ b/plugin-yarn/conf/ranger-yarn-audit-changes.cfg @@ -31,3 +31,15 @@ xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.local.archive.directory %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists + +#xasecure.audit.kafka.is.enabled %XAAUDIT.KAFKA.IS_ENABLED% mod create-if-not-exists +#xasecure.audit.kafka.is.async %XAAUDIT.KAFKA.IS_ASYNC% mod create-if-not-exists +#xasecure.audit.kafka.async.max.queue.size %XAAUDIT.KAFKA.MAX_QUEUE_SIZE% mod create-if-not-exists +#xasecure.audit.kafka.async.max.flush.interval.ms %XAAUDIT.KAFKA.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists +#xasecure.audit.kafka.broker_list %XAAUDIT.KAFKA.BROKER_LIST% mod create-if-not-exists +#xasecure.audit.kafka.topic_name %XAAUDIT.KAFKA.TOPIC_NAME% mod create-if-not-exists + +xasecure.audit.solr.is.enabled %XAAUDIT.SOLR.IS_ENABLED% mod create-if-not-exists +xasecure.audit.solr.async.max.queue.size %XAAUDIT.SOLR.MAX_QUEUE_SIZE% mod create-if-not-exists +xasecure.audit.solr.async.max.flush.interval.ms %XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists +xasecure.audit.solr.solr_url %XAAUDIT.SOLR.SOLR_URL% mod create-if-not-exists http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/plugin-yarn/conf/ranger-yarn-audit.xml ---------------------------------------------------------------------- diff --git a/plugin-yarn/conf/ranger-yarn-audit.xml b/plugin-yarn/conf/ranger-yarn-audit.xml index c0096a4..f1e9687 100644 --- a/plugin-yarn/conf/ranger-yarn-audit.xml +++ b/plugin-yarn/conf/ranger-yarn-audit.xml @@ -184,4 +184,52 @@ <name>xasecure.audit.log4j.async.max.flush.interval.ms</name> <value>30000</value> </property> + + <!-- Kafka audit provider configuration --> + <property> + <name>xasecure.audit.kafka.is.enabled</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.kafka.async.max.queue.size</name> + <value>1</value> + </property> + + <property> + <name>xasecure.audit.kafka.async.max.flush.interval.ms</name> + <value>1000</value> + </property> + + <property> + <name>xasecure.audit.kafka.broker_list</name> + <value>localhost:9092</value> + </property> + + <property> + <name>xasecure.audit.kafka.topic_name</name> + <value>ranger_audits</value> + </property> + + <!-- Ranger audit provider configuration --> + <property> + <name>xasecure.audit.ranger.is.enabled</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.ranger.async.max.queue.size</name> + <value>1</value> + </property> + + <property> + <name>xasecure.audit.ranger.async.max.flush.interval.ms</name> + <value>1000</value> + </property> + + <property> + <name>xasecure.audit.solr.solr_url</name> + <value>http://localhost:6083/solr/ranger_audits</value> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/plugin-yarn/scripts/install.properties ---------------------------------------------------------------------- diff --git a/plugin-yarn/scripts/install.properties b/plugin-yarn/scripts/install.properties index d2d1ffe..bbe9f7f 100644 --- a/plugin-yarn/scripts/install.properties +++ b/plugin-yarn/scripts/install.properties @@ -89,6 +89,12 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 +#Solr Audit Provder +XAAUDIT.SOLR.IS_ENABLED=false +XAAUDIT.SOLR.MAX_QUEUE_SIZE=1 +XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000 +XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits + # # SSL Client Certificate Information # http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index ef39d37..00c8339 100644 --- a/pom.xml +++ b/pom.xml @@ -88,6 +88,7 @@ <module>knox-agent</module> <module>storm-agent</module> <module>plugin-yarn</module> + <module>ranger_solrj</module> <module>security-admin</module> <module>ugsync</module> <module>unixauthclient</module> @@ -129,6 +130,9 @@ <hamcrest.all.version>1.3</hamcrest.all.version> <hbase.version>0.99.2</hbase.version> <hive.version>1.2.0-SNAPSHOT</hive.version> + <httpcomponent.httpmime.version>4.2.5</httpcomponent.httpmime.version> + <httpcomponent.httpclient.version>4.2.5</httpcomponent.httpclient.version> + <httpcomponent.httpcore.version>4.2.5</httpcomponent.httpcore.version> <calcite.version>0.9.2-incubating</calcite.version> <tez.version>0.5.2</tez.version> <javassist.version>3.12.1.GA</javassist.version> @@ -138,16 +142,20 @@ <jersey-bundle.version>1.17.1</jersey-bundle.version> <jersey-client.version>2.6</jersey-client.version> <junit.version>4.11</junit.version> + <kafka.version>0.8.2.0</kafka.version> <mockito.version>1.8.4</mockito.version> <hamcrest-version>1.3</hamcrest-version> <knox.gateway.version>0.5.0</knox.gateway.version> <local.lib.dir>${project.basedir}/../lib/local</local.lib.dir> <log4j.version>1.2.17</log4j.version> <mysql-connector-java.version>5.1.31</mysql-connector-java.version> + <noggit.version>0.6</noggit.version> <owasp-java-html-sanitizer.version>r239</owasp-java-html-sanitizer.version> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <security-agent-install-dir>hadoop-security/plugins</security-agent-install-dir> <slf4j-api.version>1.7.5</slf4j-api.version> + <!--<solr.version>5.0.0</solr.version>--> + <ranger.solrj.version>0.4.0</ranger.solrj.version> <springframework.spring.version>2.5.6</springframework.spring.version> <!-- <springframework.spring.version>3.1.3.RELEASE</springframework.spring.version> @@ -162,6 +170,7 @@ <tomcat.commons.el.version>5.5.23</tomcat.commons.el.version> <tomcat.embed.version>7.0.55</tomcat.embed.version> <velocity.version>1.7</velocity.version> + <zookeeper.version>3.4.6</zookeeper.version> <powermock.version>1.5.6</powermock.version> <aspectj.version>1.8.2</aspectj.version> <findbugs.plugin.version>3.0.0</findbugs.plugin.version> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/.gitignore ---------------------------------------------------------------------- diff --git a/ranger_solrj/.gitignore b/ranger_solrj/.gitignore new file mode 100644 index 0000000..b83d222 --- /dev/null +++ b/ranger_solrj/.gitignore @@ -0,0 +1 @@ +/target/ http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/pom.xml ---------------------------------------------------------------------- diff --git a/ranger_solrj/pom.xml b/ranger_solrj/pom.xml new file mode 100644 index 0000000..2b86140 --- /dev/null +++ b/ranger_solrj/pom.xml @@ -0,0 +1,55 @@ +<?xml version="1.0"?> +<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"; xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.ranger</groupId> + <artifactId>ranger</artifactId> + <version>0.4.0</version> + </parent> + <groupId>org.apache.ranger</groupId> + <artifactId>ranger_solrj</artifactId> + <version>0.4.0</version> + <name>ranger_solrj</name> + <url>http://maven.apache.org</url> + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + </properties> + <dependencies> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + <version>${commons.io.version}</version> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + <version>${httpcomponent.httpclient.version}</version> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpcore</artifactId> + <version>${httpcomponent.httpcore.version}</version> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpmime</artifactId> + <version>${httpcomponent.httpmime.version}</version> + </dependency> + <dependency> + <groupId>org.noggit</groupId> + <artifactId>noggit</artifactId> + <version>${noggit.version}</version> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + <version>${slf4j-api.version}</version> + </dependency> + <dependency> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + <version>${zookeeper.version}</version> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/ResponseParser.java ---------------------------------------------------------------------- diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/ResponseParser.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/ResponseParser.java new file mode 100644 index 0000000..d5c3b38 --- /dev/null +++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/ResponseParser.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.client.solrj; + +import java.io.Reader; +import java.io.InputStream; +import org.apache.solr.common.util.NamedList; + +/** + * + * + * @since solr 1.3 + */ +public abstract class ResponseParser +{ + public abstract String getWriterType(); // for example: wt=XML, JSON, etc + + public abstract NamedList<Object> processResponse(InputStream body, String encoding); + + public abstract NamedList<Object> processResponse(Reader reader); + + /** + * A well behaved ResponseParser will return its content-type. + * + * @return the content-type this parser expects to parse + */ + public String getContentType() { + return null; + } + + /** + * @return the version param passed to solr + */ + public String getVersion() + { + return "2.2"; + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrClient.java ---------------------------------------------------------------------- diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrClient.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrClient.java new file mode 100644 index 0000000..28b7f4f --- /dev/null +++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrClient.java @@ -0,0 +1,416 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.client.solrj; + +import org.apache.solr.client.solrj.SolrRequest.METHOD; +import org.apache.solr.client.solrj.beans.DocumentObjectBinder; +import org.apache.solr.client.solrj.impl.StreamingBinaryResponseParser; +import org.apache.solr.client.solrj.request.QueryRequest; +import org.apache.solr.client.solrj.request.SolrPing; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.SolrPingResponse; +import org.apache.solr.client.solrj.response.UpdateResponse; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.StringUtils; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; + +import java.io.Closeable; +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +/** + * Abstraction through which all communication with a Solr server may be routed + * + * @since 5.0, replaced {@code SolrServer} + */ +public abstract class SolrClient implements Serializable, Closeable { + + private static final long serialVersionUID = 1L; + private DocumentObjectBinder binder; + + /** + * Adds a collection of documents + * @param docs the collection of documents + * @throws IOException If there is a low-level I/O error. + */ + public UpdateResponse add(Collection<SolrInputDocument> docs) throws SolrServerException, IOException { + return add(docs, -1); + } + + /** + * Adds a collection of documents, specifying max time before they become committed + * @param docs the collection of documents + * @param commitWithinMs max time (in ms) before a commit will happen + * @throws IOException If there is a low-level I/O error. + * @since solr 3.5 + */ + public UpdateResponse add(Collection<SolrInputDocument> docs, int commitWithinMs) throws SolrServerException, IOException { + UpdateRequest req = new UpdateRequest(); + req.add(docs); + req.setCommitWithin(commitWithinMs); + return req.process(this); + } + + /** + * Adds a collection of beans + * @param beans the collection of beans + * @throws IOException If there is a low-level I/O error. + */ + public UpdateResponse addBeans(Collection<?> beans) throws SolrServerException, IOException { + return addBeans(beans, -1); + } + + /** + * Adds a collection of beans specifying max time before they become committed + * @param beans the collection of beans + * @param commitWithinMs max time (in ms) before a commit will happen + * @throws IOException If there is a low-level I/O error. + * @since solr 3.5 + */ + public UpdateResponse addBeans(Collection<?> beans, int commitWithinMs) throws SolrServerException, IOException { + DocumentObjectBinder binder = this.getBinder(); + ArrayList<SolrInputDocument> docs = new ArrayList<>(beans.size()); + for (Object bean : beans) { + docs.add(binder.toSolrInputDocument(bean)); + } + return add(docs, commitWithinMs); + } + + /** + * Adds a single document + * @param doc the input document + * @throws IOException If there is a low-level I/O error. + */ + public UpdateResponse add(SolrInputDocument doc) throws SolrServerException, IOException { + return add(doc, -1); + } + + /** + * Adds a single document specifying max time before it becomes committed + * @param doc the input document + * @param commitWithinMs max time (in ms) before a commit will happen + * @throws IOException If there is a low-level I/O error. + * @since solr 3.5 + */ + public UpdateResponse add(SolrInputDocument doc, int commitWithinMs) throws SolrServerException, IOException { + UpdateRequest req = new UpdateRequest(); + req.add(doc); + req.setCommitWithin(commitWithinMs); + return req.process(this); + } + + /** + * Adds a single bean + * @param obj the input bean + * @throws IOException If there is a low-level I/O error. + */ + public UpdateResponse addBean(Object obj) throws IOException, SolrServerException { + return addBean(obj, -1); + } + + /** + * Adds a single bean specifying max time before it becomes committed + * @param obj the input bean + * @param commitWithinMs max time (in ms) before a commit will happen + * @throws IOException If there is a low-level I/O error. + * @since solr 3.5 + */ + public UpdateResponse addBean(Object obj, int commitWithinMs) throws IOException, SolrServerException { + return add(getBinder().toSolrInputDocument(obj),commitWithinMs); + } + + /** + * Performs an explicit commit, causing pending documents to be committed for indexing + * <p> + * waitFlush=true and waitSearcher=true to be inline with the defaults for plain HTTP access + * @throws IOException If there is a low-level I/O error. + */ + public UpdateResponse commit() throws SolrServerException, IOException { + return commit(true, true); + } + + /** + * Performs an explicit optimize, causing a merge of all segments to one. + * <p> + * waitFlush=true and waitSearcher=true to be inline with the defaults for plain HTTP access + * <p> + * Note: In most cases it is not required to do explicit optimize + * @throws IOException If there is a low-level I/O error. + */ + public UpdateResponse optimize() throws SolrServerException, IOException { + return optimize(true, true, 1); + } + + /** + * Performs an explicit commit, causing pending documents to be committed for indexing + * @param waitFlush block until index changes are flushed to disk + * @param waitSearcher block until a new searcher is opened and registered as the main query searcher, making the changes visible + * @throws IOException If there is a low-level I/O error. + */ + public UpdateResponse commit(boolean waitFlush, boolean waitSearcher) throws SolrServerException, IOException { + return new UpdateRequest().setAction(UpdateRequest.ACTION.COMMIT, waitFlush, waitSearcher).process( this ); + } + + /** + * Performs an explicit commit, causing pending documents to be committed for indexing + * @param waitFlush block until index changes are flushed to disk + * @param waitSearcher block until a new searcher is opened and registered as the main query searcher, making the changes visible + * @param softCommit makes index changes visible while neither fsync-ing index files nor writing a new index descriptor + * @throws IOException If there is a low-level I/O error. + */ + public UpdateResponse commit(boolean waitFlush, boolean waitSearcher, boolean softCommit) throws SolrServerException, IOException { + return new UpdateRequest().setAction(UpdateRequest.ACTION.COMMIT, waitFlush, waitSearcher, softCommit).process( this ); + } + + /** + * Performs an explicit optimize, causing a merge of all segments to one. + * <p> + * Note: In most cases it is not required to do explicit optimize + * @param waitFlush block until index changes are flushed to disk + * @param waitSearcher block until a new searcher is opened and registered as the main query searcher, making the changes visible + * @throws IOException If there is a low-level I/O error. + */ + public UpdateResponse optimize(boolean waitFlush, boolean waitSearcher) throws SolrServerException, IOException { + return optimize(waitFlush, waitSearcher, 1); + } + + /** + * Performs an explicit optimize, causing a merge of all segments to one. + * <p> + * Note: In most cases it is not required to do explicit optimize + * @param waitFlush block until index changes are flushed to disk + * @param waitSearcher block until a new searcher is opened and registered as the main query searcher, making the changes visible + * @param maxSegments optimizes down to at most this number of segments + * @throws IOException If there is a low-level I/O error. + */ + public UpdateResponse optimize(boolean waitFlush, boolean waitSearcher, int maxSegments) throws SolrServerException, IOException { + return new UpdateRequest().setAction(UpdateRequest.ACTION.OPTIMIZE, waitFlush, waitSearcher, maxSegments).process( this ); + } + + /** + * Performs a rollback of all non-committed documents pending. + * <p> + * Note that this is not a true rollback as in databases. Content you have previously + * added may have been committed due to autoCommit, buffer full, other client performing + * a commit etc. + * @throws IOException If there is a low-level I/O error. + */ + public UpdateResponse rollback() throws SolrServerException, IOException { + return new UpdateRequest().rollback().process( this ); + } + + /** + * Deletes a single document by unique ID + * @param id the ID of the document to delete + * @throws IOException If there is a low-level I/O error. + */ + public UpdateResponse deleteById(String id) throws SolrServerException, IOException { + return deleteById(id, -1); + } + + /** + * Deletes a single document by unique ID, specifying max time before commit + * @param id the ID of the document to delete + * @param commitWithinMs max time (in ms) before a commit will happen + * @throws IOException If there is a low-level I/O error. + * @since 3.6 + */ + public UpdateResponse deleteById(String id, int commitWithinMs) throws SolrServerException, IOException { + UpdateRequest req = new UpdateRequest(); + req.deleteById(id); + req.setCommitWithin(commitWithinMs); + return req.process(this); + } + + /** + * Deletes a list of documents by unique ID + * @param ids the list of document IDs to delete + * @throws IOException If there is a low-level I/O error. + */ + public UpdateResponse deleteById(List<String> ids) throws SolrServerException, IOException { + return deleteById(ids, -1); + } + + /** + * Deletes a list of documents by unique ID, specifying max time before commit + * @param ids the list of document IDs to delete + * @param commitWithinMs max time (in ms) before a commit will happen + * @throws IOException If there is a low-level I/O error. + * @since 3.6 + */ + public UpdateResponse deleteById(List<String> ids, int commitWithinMs) throws SolrServerException, IOException { + UpdateRequest req = new UpdateRequest(); + req.deleteById(ids); + req.setCommitWithin(commitWithinMs); + return req.process(this); + } + + /** + * Deletes documents from the index based on a query + * @param query the query expressing what documents to delete + * @throws IOException If there is a low-level I/O error. + */ + public UpdateResponse deleteByQuery(String query) throws SolrServerException, IOException { + return deleteByQuery(query, -1); + } + + /** + * Deletes documents from the index based on a query, specifying max time before commit + * @param query the query expressing what documents to delete + * @param commitWithinMs max time (in ms) before a commit will happen + * @throws IOException If there is a low-level I/O error. + * @since 3.6 + */ + public UpdateResponse deleteByQuery(String query, int commitWithinMs) throws SolrServerException, IOException { + UpdateRequest req = new UpdateRequest(); + req.deleteByQuery(query); + req.setCommitWithin(commitWithinMs); + return req.process(this); + } + + /** + * Issues a ping request to check if the server is alive + * @throws IOException If there is a low-level I/O error. + */ + public SolrPingResponse ping() throws SolrServerException, IOException { + return new SolrPing().process(this); + } + + /** + * Performs a query to the Solr server + * @param params an object holding all key/value parameters to send along the request + */ + public QueryResponse query(SolrParams params) throws SolrServerException, IOException { + return new QueryRequest(params).process(this); + } + + /** + * Performs a query to the Solr server + * @param params an object holding all key/value parameters to send along the request + * @param method specifies the HTTP method to use for the request, such as GET or POST + */ + public QueryResponse query(SolrParams params, METHOD method) throws SolrServerException, IOException { + return new QueryRequest(params, method).process(this); + } + + /** + * Query solr, and stream the results. Unlike the standard query, this will + * send events for each Document rather then add them to the QueryResponse. + * + * Although this function returns a 'QueryResponse' it should be used with care + * since it excludes anything that was passed to callback. Also note that + * future version may pass even more info to the callback and may not return + * the results in the QueryResponse. + * + * @since solr 4.0 + */ + public QueryResponse queryAndStreamResponse(SolrParams params, StreamingResponseCallback callback) throws SolrServerException, IOException + { + ResponseParser parser = new StreamingBinaryResponseParser(callback); + QueryRequest req = new QueryRequest(params); + req.setStreamingResponseCallback(callback); + req.setResponseParser(parser); + return req.process(this); + } + + /** + * Retrieves the SolrDocument associated with the given identifier. + * + * @return retrieved SolrDocument, null if no document is found. + */ + public SolrDocument getById(String id) throws SolrServerException, IOException { + return getById(id, null); + } + + /** + * Retrieves the SolrDocument associated with the given identifier and uses + * the SolrParams to execute the request. + * + * @return retrieved SolrDocument, null if no document is found. + */ + public SolrDocument getById(String id, SolrParams params) throws SolrServerException, IOException { + SolrDocumentList docs = getById(Arrays.asList(id), params); + if (!docs.isEmpty()) { + return docs.get(0); + } + return null; + } + + /** + * Retrieves the SolrDocuments associated with the given identifiers. + * If a document was not found, it will not be added to the SolrDocumentList. + */ + public SolrDocumentList getById(Collection<String> ids) throws SolrServerException, IOException { + return getById(ids, null); + } + + /** + * Retrieves the SolrDocuments associated with the given identifiers and uses + * the SolrParams to execute the request. + * If a document was not found, it will not be added to the SolrDocumentList. + */ + public SolrDocumentList getById(Collection<String> ids, SolrParams params) throws SolrServerException, IOException { + if (ids == null || ids.isEmpty()) { + throw new IllegalArgumentException("Must provide an identifier of a document to retrieve."); + } + + ModifiableSolrParams reqParams = new ModifiableSolrParams(params); + if (StringUtils.isEmpty(reqParams.get(CommonParams.QT))) { + reqParams.set(CommonParams.QT, "/get"); + } + reqParams.set("ids", (String[]) ids.toArray()); + + return query(reqParams).getResults(); + } + + /** + * SolrServer implementations need to implement how a request is actually processed + */ + public abstract NamedList<Object> request(final SolrRequest request) throws SolrServerException, IOException; + + public DocumentObjectBinder getBinder() { + if(binder == null){ + binder = new DocumentObjectBinder(); + } + return binder; + } + + /** + * Release allocated resources. + * + * @since solr 4.0 + * @deprecated Use close() instead. + */ + @Deprecated + public abstract void shutdown(); + + //@SuppressWarnings("deprecation") + public void close() throws IOException { + shutdown(); + } +}
