This is an automated email from the ASF dual-hosted git repository.
mehul pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new 6edc9bf RANGER-2324 : Bootstrapping Solr in Ranger service start-up
6edc9bf is described below
commit 6edc9bf03f9ef26ce14ac9c039fc1a095c71773b
Author: Bhavik Patel <[email protected]>
AuthorDate: Thu Mar 28 21:36:40 2019 +0530
RANGER-2324 : Bootstrapping Solr in Ranger service start-up
Signed-off-by: Mehul Parikh <[email protected]>
---
.../ranger/server/tomcat/EmbeddedServer.java | 11 +
.../server/tomcat/SolrCollectionBoostrapper.java | 624 +++++++++++++++++++++
security-admin/scripts/install.properties | 11 +-
security-admin/scripts/setup.sh | 51 ++
.../conf.dist/ranger-admin-default-site.xml | 5 +
.../main/resources/conf.dist/ranger-admin-site.xml | 25 +
src/main/assembly/admin-web.xml | 7 +
7 files changed, 733 insertions(+), 1 deletion(-)
diff --git
a/embeddedwebserver/src/main/java/org/apache/ranger/server/tomcat/EmbeddedServer.java
b/embeddedwebserver/src/main/java/org/apache/ranger/server/tomcat/EmbeddedServer.java
index 8d32352..8633e00 100644
---
a/embeddedwebserver/src/main/java/org/apache/ranger/server/tomcat/EmbeddedServer.java
+++
b/embeddedwebserver/src/main/java/org/apache/ranger/server/tomcat/EmbeddedServer.java
@@ -268,6 +268,17 @@ public class EmbeddedServer {
private void startServer(final Tomcat server) {
try {
+
+ try {
+ String servername = getConfig("servername");
+ LOG.info("Server Name : " + servername);
+ if
(servername.equalsIgnoreCase(ADMIN_SERVER_NAME)) {
+ SolrCollectionBoostrapper solrSetup =
new SolrCollectionBoostrapper();
+ solrSetup.start();
+ }
+ } catch (Exception e) {
+ LOG.severe("Error while setting solr " + e);
+ }
server.start();
server.getServer().await();
shutdownServer();
diff --git
a/embeddedwebserver/src/main/java/org/apache/ranger/server/tomcat/SolrCollectionBoostrapper.java
b/embeddedwebserver/src/main/java/org/apache/ranger/server/tomcat/SolrCollectionBoostrapper.java
new file mode 100644
index 0000000..0d85e91
--- /dev/null
+++
b/embeddedwebserver/src/main/java/org/apache/ranger/server/tomcat/SolrCollectionBoostrapper.java
@@ -0,0 +1,624 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ranger.server.tomcat;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.Properties;
+import java.util.UUID;
+import java.util.logging.Logger;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.security.SecureClientLogin;
+import org.apache.ranger.authorization.utils.StringUtil;
+import org.apache.ranger.plugin.util.XMLUtils;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.HttpClientUtil;
+import org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder;
+import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.SolrZooKeeper;
+import org.apache.solr.common.cloud.ZkConfigManager;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Id;
+import org.apache.zookeeper.data.Stat;
+import com.google.protobuf.TextFormat.ParseException;
+
+public class SolrCollectionBoostrapper extends Thread {
+
+ private static final Logger logger = Logger
+ .getLogger(SolrCollectionBoostrapper.class.getName());
+ final static String SOLR_ZK_HOSTS = "ranger.audit.solr.zookeepers";
+ final static String SOLR_COLLECTION_NAME =
"ranger.audit.solr.collection.name";
+ final static String SOLR_CONFIG_NAME = "ranger.audit.solr.config.name";
+ final static String SOLR_NO_SHARDS = "ranger.audit.solr.no.shards";
+ final static String SOLR_MAX_SHARD_PER_NODE =
"ranger.audit.solr.max.shards.per.node";
+ final static String SOLR_NO_REPLICA = "ranger.audit.solr.no.replica";
+ final static String SOLR_TIME_INTERVAL =
"ranger.audit.solr.time.interval";
+ final static String SOLR_ACL_USER_LIST_SASL =
"ranger.audit.solr.acl.user.list.sasl";
+ final static String PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG =
"java.security.auth.login.config";
+ public static final String DEFAULT_COLLECTION_NAME = "ranger_audits";
+ public static final String DEFAULT_CONFIG_NAME = "ranger_audits";
+ public static final String DEFAULT_SERVICE_NAME = "rangeradmin";
+ public static final long DEFAULT_SOLR_TIME_INTERVAL_MS = 60000L;
+ private static final String CONFIG_FILE = "ranger-admin-site.xml";
+ private static final String CORE_SITE_CONFIG_FILENAME = "core-site.xml";
+ private static final String DEFAULT_CONFIG_FILENAME =
"ranger-admin-default-site.xml";
+ private static final String AUTH_TYPE_KERBEROS = "kerberos";
+ private static final String AUTHENTICATION_TYPE =
"hadoop.security.authentication";
+ private static final String RANGER_SERVICE_HOSTNAME =
"ranger.service.host";
+ private static final String ADMIN_USER_PRINCIPAL =
"ranger.admin.kerberos.principal";
+ private static final String SOLR_CONFIG_FILE = "solrconfig.xml";
+ private static final String[] configFiles = { "admin-extra.html",
+ "admin-extra.menu-bottom.html",
"admin-extra.menu-top.html",
+ "elevate.xml", "enumsConfig.xml", "managed-schema",
+ "solrconfig.xml" };
+ private File configSetFolder = null;
+ private boolean hasEnumConfig;
+ boolean solr_cloud_mode = false;
+ boolean is_completed = false;
+ boolean isKERBEROS = false;
+ String principal = null;
+ String hostName;
+ String keytab;
+ String nameRules;
+ String solr_collection_name;
+ String solr_config_name;
+ Path path_for_cloud_mode;
+ int no_of_replicas;
+ int no_of_shards;
+ int max_node_per_shards;
+ Long time_interval;
+ SolrClient solrClient = null;
+ CloudSolrClient solrCloudClient = null;
+ SolrZooKeeper solrZookeeper = null;
+ SolrZkClient zkClient = null;
+
+ private Properties serverConfigProperties = new Properties();
+
+ public SolrCollectionBoostrapper() throws IOException {
+ logger.info("Starting Solr Setup");
+ XMLUtils.loadConfig(DEFAULT_CONFIG_FILENAME,
serverConfigProperties);
+ XMLUtils.loadConfig(CORE_SITE_CONFIG_FILENAME,
serverConfigProperties);
+ XMLUtils.loadConfig(CONFIG_FILE, serverConfigProperties);
+
+ logger.info("AUTHENTICATION_TYPE : " +
getConfig(AUTHENTICATION_TYPE));
+ if (getConfig(AUTHENTICATION_TYPE) != null
+ &&
getConfig(AUTHENTICATION_TYPE).trim().equalsIgnoreCase(
+ AUTH_TYPE_KERBEROS)) {
+ isKERBEROS = true;
+ hostName = getConfig(RANGER_SERVICE_HOSTNAME);
+ try {
+ principal = SecureClientLogin.getPrincipal(
+
getConfig(ADMIN_USER_PRINCIPAL), hostName);
+ } catch (IOException ignored) {
+ logger.warning("Failed to get
ranger.admin.kerberos.principal. Reason: "
+ + ignored.toString());
+ }
+ }
+
+ solr_collection_name = getConfig(SOLR_COLLECTION_NAME,
+ DEFAULT_COLLECTION_NAME);
+ logger.info("Solr Collection name provided is : "
+ + solr_collection_name);
+ solr_config_name = getConfig(SOLR_CONFIG_NAME,
DEFAULT_CONFIG_NAME);
+ logger.info("Solr Config name provided is : " +
solr_config_name);
+ no_of_replicas = getIntConfig(SOLR_NO_REPLICA, 1);
+ logger.info("No. of replicas provided is : " + no_of_replicas);
+
+ no_of_shards = getIntConfig(SOLR_NO_SHARDS, 1);
+ logger.info("No. of shards provided is : " + no_of_shards);
+ max_node_per_shards = getIntConfig(SOLR_MAX_SHARD_PER_NODE, 1);
+ logger.info("Max no of nodes per shards provided is : "
+ + max_node_per_shards);
+
+ time_interval = getLongConfig(SOLR_TIME_INTERVAL,
+ DEFAULT_SOLR_TIME_INTERVAL_MS);
+ logger.info("Solr time interval provided is : " +
time_interval);
+ if (System.getProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG) ==
null) {
+ System.setProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG,
+ "/dev/null");
+ }
+
+ String basedir = new File(".").getCanonicalPath();
+ String solrFileDir = new File(basedir).getParent();
+
+ path_for_cloud_mode = Paths.get(solrFileDir, "contrib",
+ "solr_for_audit_setup", "conf");
+ configSetFolder = path_for_cloud_mode.toFile();
+ }
+
+ public void run() {
+ logger.info("Started run method");
+
+ String zkHosts = "";
+ List<String> zookeeperHosts = null;
+ if (getConfig(SOLR_ZK_HOSTS) != null
+ &&
!StringUtil.isEmpty(getConfig(SOLR_ZK_HOSTS))) {
+ zkHosts = getConfig(SOLR_ZK_HOSTS).trim();
+ zookeeperHosts = new
ArrayList<String>(Arrays.asList(zkHosts
+ .split(",")));
+ }
+ if (zookeeperHosts != null
+ && !zookeeperHosts.isEmpty()
+ && zookeeperHosts.stream().noneMatch(
+ h ->
h.equalsIgnoreCase("none"))) {
+ logger.info("Solr zkHosts=" + zkHosts + ",
collectionName="
+ + solr_collection_name);
+ while (!is_completed) {
+ try {
+ if (connect(zookeeperHosts)) {
+ if (solr_cloud_mode) {
+ if
(uploadConfiguration() && createCollection()
+ &&
setupACL(zkClient)) {
+ is_completed =
true;
+ break;
+ } else {
+
logErrorMessageAndWait(
+
"Error while performing operations on solr. ",
+
null);
+ }
+ }
+
+ } else {
+ logErrorMessageAndWait(
+ "Cannot connect
to solr kindly check the solr related configs. ",
+ null);
+ }
+ } catch (Exception ex) {
+ logErrorMessageAndWait("Error while
configuring solr. ", ex);
+ }
+
+ }
+
+ } else {
+ logger.severe("Solr ZKHosts for Audit are empty. Please
set property "
+ + SOLR_ZK_HOSTS);
+ }
+
+ }
+
+ private boolean connect(List<String> zookeeperHosts) {
+ try {
+ logger.info("Solr is in Cloud mode");
+ if (isKERBEROS) {
+ setHttpClientBuilderForKrb();
+ }
+ solrCloudClient = new
CloudSolrClient.Builder(zookeeperHosts,
+ Optional.empty()).build();
+
solrCloudClient.setDefaultCollection(solr_collection_name);
+ solrClient = solrCloudClient;
+ solr_cloud_mode = true;
+
+ return true;
+ } catch (Exception ex) {
+ logger.severe("Can't connect to Solr server.
ZooKeepers="
+ + zookeeperHosts + ", collection=" +
solr_collection_name
+ + ex);
+ return false;
+ }
+ }
+
+ private void setHttpClientBuilderForKrb() {
+ Krb5HttpClientBuilder krbBuild = new Krb5HttpClientBuilder();
+ SolrHttpClientBuilder kb = krbBuild.getBuilder();
+ HttpClientUtil.setHttpClientBuilder(kb);
+ }
+
+ private boolean uploadConfiguration() {
+ try {
+ Path path =
Paths.get(System.getProperty("java.io.tmpdir"), UUID
+ .randomUUID().toString(),
"RangerAudit");
+
+ File tmpDir = path.toFile();
+
+ solrCloudClient.connect();
+ zkClient =
solrCloudClient.getZkStateReader().getZkClient();
+
+ if (zkClient != null) {
+ ZkConfigManager zkConfigManager = new
ZkConfigManager(zkClient);
+
+ boolean configExists = zkConfigManager
+ .configExists(solr_config_name);
+ if (!configExists) {
+ logger.info("Config does not exist with
name "
+ + solr_config_name);
+ doIfConfigNotExist(solr_config_name,
zkConfigManager);
+ uploadMissingConfigFiles(zkClient,
zkConfigManager,
+ solr_config_name);
+ return true;
+ } else {
+ logger.info("Config exist with name " +
solr_config_name);
+ doIfConfigExists(solr_config_name,
zkClient, path, tmpDir);
+ return true;
+ }
+ } else {
+ logger.severe("Solr is in cloud mode and could
not find the zookeeper client for performing upload operations. ");
+ return false;
+ }
+ } catch (Exception ex) {
+ logger.severe("Error while uploading configuration : "
+ ex);
+ return false;
+ }
+
+ }
+
+ private void doIfConfigNotExist(String configName,
+ ZkConfigManager zkConfigManager) throws IOException {
+ File[] listOfFiles = getConfigSetFolder().listFiles();
+ if (listOfFiles != null) {
+
zkConfigManager.uploadConfigDir(getConfigSetFolder().toPath(),
+ configName);
+ }
+ }
+
+ private void doIfConfigExists(String solrConfigName, SolrZkClient
zkClient,
+ Path path, File tmpDir) throws IOException {
+
+ logger.info("Config set exists for " + solr_collection_name
+ + " collection. Refreshing it if needed...");
+ if (!tmpDir.mkdirs()) {
+ logger.severe("Cannot create directories for"
+ + tmpDir.getAbsolutePath());
+ }
+ ZkConfigManager zkConfigManager = new ZkConfigManager(zkClient);
+ zkConfigManager.downloadConfigDir(solrConfigName, path);
+ File[] listOfFiles = getConfigSetFolder().listFiles();
+ if (listOfFiles != null) {
+ for (File file : listOfFiles) {
+ if (file.getName().equals(getConfigFileName())
+ &&
updateConfigIfNeeded(solrConfigName, zkClient, file,
+ path)) {
+ break;
+ }
+ }
+ }
+ }
+
+ private void uploadMissingConfigFiles(SolrZkClient zkClient,
+ ZkConfigManager zkConfigManager, String configName)
+ throws IOException {
+ logger.info("Check any of the configs files are missing for
config"
+ + configName);
+
+ for (String configFile : configFiles) {
+ if ("enumsConfig.xml".equals(configFile) &&
!hasEnumConfig) {
+ logger.info("Config file " + configFile + " is
not needed for "
+ + configName);
+ continue;
+ }
+ Path zkPath = Paths.get(configName, configFile);
+ if (zkConfigManager.configExists(zkPath.toString())) {
+ logger.info("Config file " + configFile
+ + " has already uploaded
properly.");
+ } else {
+ logger.info("Config file " + configFile
+ + " is missing. Reupload...");
+ FileSystems.getDefault().getSeparator();
+ uploadFileToZk(zkClient,
+
Paths.get(getConfigSetFolder().toString(), configFile),
+ Paths.get("/configs/",
zkPath.toString()));
+
+ }
+ }
+ }
+
+ private boolean updateConfigIfNeeded(String solrConfigName,
+ SolrZkClient zkClient, File file, Path path) throws
IOException {
+ boolean result = false;
+ if (!FileUtils.contentEquals(file,
+ Paths.get(path.toString(),
file.getName()).toFile())) {
+ logger.info("Solr config file differs " + file.getName()
+ + " upload config set to zookeeper");
+ Path filePath =
Paths.get(getConfigSetFolder().toString(),
+ getConfigFileName());
+ Path configsPath = Paths.get("/configs", solrConfigName,
+ getConfigFileName());
+ uploadFileToZk(zkClient, filePath, configsPath);
+ result = true;
+ } else {
+ logger.info("Content is same of SolrConfig file");
+ }
+ return result;
+ }
+
+ private void logErrorMessageAndWait(String msg, Exception exception) {
+ if (exception != null) {
+ logger.severe(msg + " [retrying after " + time_interval
+ + " ms]. Error : " + exception);
+ } else {
+ logger.severe(msg + " [retrying after " + time_interval
+ " ms]");
+ }
+
+ try {
+ Thread.sleep(time_interval);
+ } catch (InterruptedException ex) {
+ logger.info("sleep interrupted: " + ex.getMessage());
+ }
+ }
+
+ private boolean createCollection() {
+ try {
+ List<String> allCollectionList = getCollections();
+ if (allCollectionList != null) {
+ if
(!allCollectionList.contains(solr_collection_name)) {
+
+ CollectionAdminRequest.Create
createCollection = CollectionAdminRequest
+
.createCollection(solr_collection_name,
+
solr_config_name, no_of_shards,
+
no_of_replicas);
+
createCollection.setMaxShardsPerNode(max_node_per_shards);
+ CollectionAdminResponse createResponse
= createCollection
+ .process(solrClient);
+ if (createResponse.getStatus() != 0) {
+ logger.severe("Error creating
collection. collectionName="
+ +
solr_collection_name
+ + " , solr
config name = "
+ +
solr_config_name
+ + " , replicas
= "
+ + no_of_replicas
+ + ", shards="
+ + no_of_shards
+ + " , max node
per shards = "
+ +
max_node_per_shards
+ + ", response="
+ +
createResponse);
+ return false;
+ } else {
+ logger.info("Created collection
"
+ +
solr_collection_name + " with config name "
+ +
solr_config_name + " replicas = "
+ +
no_of_replicas + " Shards = " + no_of_shards
+ + " max node
per shards = "
+ +
max_node_per_shards);
+ return true;
+ }
+ } else {
+ logger.info("Collection already exists
with name "
+ + solr_collection_name);
+ return true;
+ }
+ } else {
+ logger.severe("Error while connecting to solr
");
+ return false;
+ }
+ } catch (Exception ex) {
+ logger.severe("Error while creating collection in solr
: " + ex);
+ return false;
+ }
+ }
+
+ private String getConfig(String key, String defaultValue) {
+ String ret = getConfig(key);
+ if (ret == null) {
+ ret = defaultValue;
+ }
+ return ret;
+ }
+
+ private boolean setupACL(SolrZkClient zkClient) {
+ solrZookeeper = zkClient.getSolrZooKeeper();
+ String serviceName = "";
+ List<String> aclUserList = null;
+ Path zNodeConfigPath = Paths.get("/configs", solr_config_name);
+ Path zNodeCollectionPath = Paths.get("/collections",
+ solr_collection_name);
+ if (isKERBEROS) {
+ if (principal != null &&
!StringUtil.isEmpty(principal)) {
+ serviceName = principal.substring(0,
principal.indexOf("/"));
+ } else {
+ serviceName = DEFAULT_SERVICE_NAME;
+ }
+ aclUserList = new ArrayList<String>();
+ if (getConfig(SOLR_ACL_USER_LIST_SASL) != null
+ &&
!StringUtil.isEmpty(SOLR_ACL_USER_LIST_SASL)) {
+ aclUserList =
Arrays.asList(getConfig(SOLR_ACL_USER_LIST_SASL)
+ .trim().split(","));
+ }
+ }
+ if (performACLOnZnode(zNodeConfigPath, solrZookeeper,
serviceName,
+ aclUserList)
+ && performACLOnZnode(zNodeCollectionPath,
solrZookeeper,
+ serviceName, aclUserList)) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * In case of Kerberize env scheme "world" users should have only READ
+ * access and scheme "sasl" users e.g. infra-solr, solr, rangeradmin
should
+ * have all access on Znode. In case of simple env scheme "world" users
+ * should have all access on Znode.
+ */
+ private boolean performACLOnZnode(Path zookeeperNodePath,
+ SolrZooKeeper solrZookeeper, String serviceName,
+ List<String> aclUserList) {
+ List<ACL> aclListForZnodePath = new ArrayList<ACL>();
+ try {
+
+ if (isKERBEROS) {
+ Stat stat =
solrZookeeper.exists(zookeeperNodePath.toString(),
+ false);
+
+ List<ACL> existingACLOnZnodePath =
solrZookeeper.getACL(
+ zookeeperNodePath.toString(),
stat);
+ if (existingACLOnZnodePath != null
+ &&
existingACLOnZnodePath.size() > 0) {
+ List<String> existingIdForZnodePath =
new ArrayList<String>();
+ int permissionForWorldScheme = 0;
+ for (ACL acl : existingACLOnZnodePath) {
+
existingIdForZnodePath.add(acl.getId().getId());
+ if
(acl.getId().getId().equalsIgnoreCase("anyone")) {
+
permissionForWorldScheme = acl.getPerms();
+ }
+
+ }
+
+ if (aclUserList != null &&
!aclUserList.isEmpty()) {
+ for (String aclUser :
aclUserList) {
+ if
(!existingIdForZnodePath.contains(aclUser)) {
+
aclListForZnodePath.add(new ACL(
+
ZooDefs.Perms.ALL, new Id("sasl",
+
aclUser)));
+ }
+ }
+ }
+ if
(!existingIdForZnodePath.contains(serviceName)) {
+ aclListForZnodePath.add(new
ACL(ZooDefs.Perms.ALL,
+ new Id("sasl",
serviceName)));
+ }
+ if
(!existingIdForZnodePath.contains("anyone")
+ ||
(existingACLOnZnodePath.contains("anyone") && permissionForWorldScheme !=
ZooDefs.Perms.READ)) {
+ aclListForZnodePath.add(new
ACL(ZooDefs.Perms.READ,
+ new Id("world",
"anyone")));
+ }
+ } else {
+ aclListForZnodePath.add(new
ACL(ZooDefs.Perms.READ, new Id(
+ "world", "anyone")));
+ aclListForZnodePath.add(new
ACL(ZooDefs.Perms.ALL, new Id(
+ "sasl", serviceName)));
+ for (String aclUser : aclUserList) {
+ aclListForZnodePath.add(new
ACL(ZooDefs.Perms.ALL,
+ new Id("sasl",
aclUser)));
+ }
+ }
+
+ } else {
+ aclListForZnodePath.add(new
ACL(ZooDefs.Perms.ALL, new Id(
+ "world", "anyone")));
+ }
+
+ if (aclListForZnodePath != null &&
!aclListForZnodePath.isEmpty()) {
+
solrZookeeper.setACL(zookeeperNodePath.toString(),
+ aclListForZnodePath, -1);
+ }
+
+ } catch (Exception ex) {
+ logger.severe("Error while performing ACL on zNode : "
+ + zookeeperNodePath.toString() + "
Error: " + ex);
+ return false;
+ }
+ return true;
+ }
+
+ @SuppressWarnings("unchecked")
+ private List<String> getCollections() throws IOException,
ParseException {
+ try {
+ CollectionAdminRequest.List colListReq = new
CollectionAdminRequest.List();
+ CollectionAdminResponse response =
colListReq.process(solrClient);
+ if (response.getStatus() != 0) {
+ logger.severe("Error getting collection list
from solr. response="
+ + response);
+ return null;
+ }
+ return (List<String>)
response.getResponse().get("collections");
+ } catch (SolrException e) {
+ logger.severe("getCollections() operation failed : " +
e);
+ return null;
+ } catch (SolrServerException e) {
+ logger.severe("getCollections() operation failed : " +
e);
+ return null;
+ }
+
+ }
+
+ private int getIntConfig(String key, int defaultValue) {
+ int ret = defaultValue;
+ String retStr = getConfig(key);
+ try {
+ if (retStr != null) {
+ ret = Integer.parseInt(retStr);
+ }
+ } catch (Exception err) {
+ logger.severe(retStr + " can't be parsed to int.
Reason: "
+ + err.toString());
+ }
+ return ret;
+ }
+
+ private Long getLongConfig(String key, Long defaultValue) {
+ Long ret = defaultValue;
+ String retStr = getConfig(key);
+ try {
+ if (retStr != null) {
+ ret = Long.parseLong(retStr);
+ }
+ } catch (Exception err) {
+ logger.severe(retStr + " can't be parsed to long.
Reason: "
+ + err.toString());
+ }
+ return ret;
+ }
+
+ private String getConfig(String key) {
+
+ String value = serverConfigProperties.getProperty(key);
+ if (value == null || value.trim().isEmpty()) {
+ // Value not found in properties file, let's try to get
from
+ // System's property
+ value = System.getProperty(key);
+ }
+ return value;
+ }
+
+ private void uploadFileToZk(SolrZkClient zkClient, Path filePath,
+ Path configsPath) throws FileNotFoundException {
+ InputStream is = new FileInputStream(filePath.toString());
+ try {
+ if (zkClient.exists(configsPath.toString(), true)) {
+ zkClient.setData(configsPath.toString(),
+ IOUtils.toByteArray(is), true);
+ } else {
+ zkClient.create(configsPath.toString(),
+ IOUtils.toByteArray(is),
CreateMode.PERSISTENT, true);
+ }
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ } finally {
+ IOUtils.closeQuietly(is);
+ }
+ }
+
+ private File getConfigSetFolder() {
+ return configSetFolder;
+ }
+
+ private String getConfigFileName() {
+ return SOLR_CONFIG_FILE;
+ }
+}
\ No newline at end of file
diff --git a/security-admin/scripts/install.properties
b/security-admin/scripts/install.properties
index fdcee1b..155c42c 100644
--- a/security-admin/scripts/install.properties
+++ b/security-admin/scripts/install.properties
@@ -87,6 +87,15 @@ audit_solr_user=
audit_solr_password=
audit_solr_zookeepers=
+audit_solr_collection_name=ranger_audits
+#solr Properties for cloud mode
+audit_solr_config_name=ranger_audits
+audit_solr_no_shards=1
+audit_solr_no_replica=1
+audit_solr_max_shards_per_node=1
+audit_solr_acl_user_list_sasl=solr,infra-solr
+
+
#------------------------- DB CONFIG - END ----------------------------------
#
@@ -249,4 +258,4 @@ sqlserver_audit_file=db/sqlserver/xa_audit_db_sqlserver.sql
#
sqlanywhere_core_file=db/sqlanywhere/optimized/current/ranger_core_db_sqlanywhere.sql
sqlanywhere_audit_file=db/sqlanywhere/xa_audit_db_sqlanywhere.sql
-cred_keystore_filename=$app_home/WEB-INF/classes/conf/.jceks/rangeradmin.jceks
\ No newline at end of file
+cred_keystore_filename=$app_home/WEB-INF/classes/conf/.jceks/rangeradmin.jceks
diff --git a/security-admin/scripts/setup.sh b/security-admin/scripts/setup.sh
index bd4bd4c..4916828 100755
--- a/security-admin/scripts/setup.sh
+++ b/security-admin/scripts/setup.sh
@@ -144,6 +144,13 @@ admin_keytab=$(get_prop 'admin_keytab' $PROPFILE)
lookup_principal=$(get_prop 'lookup_principal' $PROPFILE)
lookup_keytab=$(get_prop 'lookup_keytab' $PROPFILE)
hadoop_conf=$(get_prop 'hadoop_conf' $PROPFILE)
+audit_solr_collection_name=$(get_prop 'audit_solr_collection_name' $PROPFILE)
+audit_solr_config_name=$(get_prop 'audit_solr_config_name' $PROPFILE)
+audit_solr_no_shards=$(get_prop 'audit_solr_no_shards' $PROPFILE)
+audit_solr_no_replica=$(get_prop 'audit_solr_no_replica' $PROPFILE)
+audit_solr_max_shards_per_node=$(get_prop 'audit_solr_max_shards_per_node'
$PROPFILE)
+audit_solr_acl_user_list_sasl=$(get_prop 'audit_solr_acl_user_list_sasl'
$PROPFILE)
+
DB_HOST="${db_host}"
@@ -422,6 +429,50 @@ update_properties() {
updatePropertyToFilePy $propertyName $newPropertyValue
$to_file_ranger
fi
+ if [ "${audit_solr_collection_name}" != "" ]
+ then
+ propertyName=ranger.audit.solr.collection.name
+ newPropertyValue="${audit_solr_collection_name}"
+ updatePropertyToFilePy $propertyName $newPropertyValue
$to_file_ranger
+ fi
+
+ if [ "${audit_solr_config_name}" != "" ]
+ then
+ propertyName=ranger.audit.solr.config.name
+ newPropertyValue="${audit_solr_config_name}"
+ updatePropertyToFilePy $propertyName $newPropertyValue
$to_file_ranger
+ fi
+
+ if [ "${audit_solr_no_shards}" != "" ]
+ then
+ propertyName=ranger.audit.solr.no.shards
+ newPropertyValue="${audit_solr_no_shards}"
+ updatePropertyToFilePy $propertyName $newPropertyValue
$to_file_ranger
+ fi
+
+
+ if [ "${audit_solr_max_shards_per_node}" != "" ]
+ then
+ propertyName=ranger.audit.solr.max.shards.per.node
+ newPropertyValue="${audit_solr_max_shards_per_node}"
+ updatePropertyToFilePy $propertyName $newPropertyValue
$to_file_ranger
+ fi
+
+ if [ "${audit_solr_no_replica}" != "" ]
+ then
+ propertyName=ranger.audit.solr.no.replica
+ newPropertyValue="${audit_solr_no_replica}"
+ updatePropertyToFilePy $propertyName $newPropertyValue
$to_file_ranger
+ fi
+
+ if [ "${audit_solr_acl_user_list_sasl}" != "" ]
+ then
+ propertyName=ranger.audit.solr.acl.user.list.sasl
+ newPropertyValue="${audit_solr_acl_user_list_sasl}"
+ updatePropertyToFilePy $propertyName $newPropertyValue
$to_file_ranger
+ fi
+
+
if [ "${spnego_keytab}" != "" ]
then
propertyName=ranger.spnego.kerberos.keytab
diff --git
a/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml
b/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml
index 686f720..189f7b8 100644
--- a/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml
+++ b/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml
@@ -436,6 +436,11 @@
<description></description>
</property>
<property>
+ <name>ranger.audit.solr.time.interval</name>
+ <value>60000</value>
+ <description>Time in milliseconds</description>
+ </property>
+ <property>
<name>ranger.sha256Password.update.disable</name>
<value>true</value>
<description></description>
diff --git a/security-admin/src/main/resources/conf.dist/ranger-admin-site.xml
b/security-admin/src/main/resources/conf.dist/ranger-admin-site.xml
index 4d4a1de..26a9d64 100644
--- a/security-admin/src/main/resources/conf.dist/ranger-admin-site.xml
+++ b/security-admin/src/main/resources/conf.dist/ranger-admin-site.xml
@@ -294,4 +294,29 @@
<name>ranger.tomcat.ciphers</name>
<value></value>
</property>
+ <!-- Solr related porps -->
+ <property>
+ <name>ranger.audit.solr.collection.name</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.audit.solr.config.name</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.audit.solr.no.shards</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.audit.solr.max.shards.per.node</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.audit.solr.no.replica</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.audit.solr.acl.user.list.sasl</name>
+ <value></value>
+ </property>
</configuration>
diff --git a/src/main/assembly/admin-web.xml b/src/main/assembly/admin-web.xml
index 4c92720..06ba8c4 100644
--- a/src/main/assembly/admin-web.xml
+++ b/src/main/assembly/admin-web.xml
@@ -213,6 +213,13 @@
<include>commons-collections:commons-collections</include>
<include>commons-lang:commons-lang</include>
<include>commons-io:commons-io</include>
+ <include>org.apache.solr:solr-solrj</include>
+ <include>org.apache.httpcomponents:httpclient</include>
+ <include>org.apache.httpcomponents:httpcore</include>
+ <include>org.noggit:noggit</include>
+ <include>org.apache.zookeeper:zookeeper</include>
+ <include>org.apache.httpcomponents:httpmime</include>
+ <include>commons-codec:commons-codec</include>
<include>org.apache.htrace:htrace-core4:jar:${htrace-core.version}</include>
<include>com.fasterxml.woodstox:woodstox-core:jar:${fasterxml.woodstox.version}</include>
<include>org.codehaus.woodstox:stax2-api:jar:${codehaus.woodstox.stax2api.version}</include>
