This is an automated email from the ASF dual-hosted git repository. pradeep pushed a commit to branch RANGER-5061_master in repository https://gitbox.apache.org/repos/asf/ranger.git
commit 9c3ac972f60bba28a6751d830b51d1ef238d484f Author: Vyom Mani Tiwari <[email protected]> AuthorDate: Sat Jan 4 00:47:39 2025 +0530 RANGER-5061: security-admin module(elasticsearch package): update for code readability improvement (#488) * RANGER-5069: Add ability to Kafka authorizer to define super users through Kafka config Co-authored-by: Daniel Fonai <[email protected]> Signed-off-by: Madhan Neethiraj <[email protected]> --- .../ElasticSearchAccessAuditsService.java | 541 +++++++++++---------- .../ranger/elasticsearch/ElasticSearchMgr.java | 220 +++++---- .../ranger/elasticsearch/ElasticSearchUtil.java | 130 +++-- 3 files changed, 486 insertions(+), 405 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java index a64cb111e..4f6c808ff 100644 --- a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java +++ b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java @@ -27,10 +27,10 @@ import org.apache.ranger.common.SearchCriteria; import org.apache.ranger.db.XXServiceDefDao; import org.apache.ranger.entity.XXService; import org.apache.ranger.entity.XXServiceDef; +import org.apache.ranger.plugin.util.JsonUtilsV2; import org.apache.ranger.view.VXAccessAudit; import org.apache.ranger.view.VXAccessAuditList; import org.apache.ranger.view.VXLong; -import org.apache.ranger.plugin.util.JsonUtilsV2; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.get.MultiGetItemResponse; import org.elasticsearch.action.search.SearchResponse; @@ -50,260 +50,289 @@ import java.util.Map; @Service @Scope("singleton") public class ElasticSearchAccessAuditsService extends org.apache.ranger.AccessAuditsService { - private static final Logger LOGGER = LoggerFactory.getLogger(ElasticSearchAccessAuditsService.class); - - @Autowired - ElasticSearchMgr elasticSearchMgr; - - @Autowired - ElasticSearchUtil elasticSearchUtil; - - - public VXAccessAuditList searchXAccessAudits(SearchCriteria searchCriteria) { - - RestHighLevelClient client = elasticSearchMgr.getClient(); - final boolean hiveQueryVisibility = PropertiesUtil.getBooleanProperty("ranger.audit.hive.query.visibility", true); - if (client == null) { - LOGGER.warn("ElasticSearch client is null, so not running the query."); - throw restErrorUtil.createRESTException( - "Error connecting to search engine", - MessageEnums.ERROR_SYSTEM); - } - List<VXAccessAudit> xAccessAuditList = new ArrayList<VXAccessAudit>(); - Map<String, Object> paramList = searchCriteria.getParamList(); - updateUserExclusion(paramList); - - SearchResponse response; - try { - response = elasticSearchUtil.searchResources(searchCriteria, searchFields, sortFields, client, elasticSearchMgr.index); - } catch (IOException e) { - LOGGER.warn(String.format("ElasticSearch query failed: %s", e.getMessage())); - throw restErrorUtil.createRESTException( - "Error querying search engine", - MessageEnums.ERROR_SYSTEM); - } - MultiGetItemResponse[] docs; - try { - docs = elasticSearchUtil.fetch(client, elasticSearchMgr.index, response.getHits().getHits()); - } catch (IOException e) { - LOGGER.warn(String.format("ElasticSearch fetch failed: %s", e.getMessage())); - throw restErrorUtil.createRESTException( - "Error querying search engine", - MessageEnums.ERROR_SYSTEM); - } - for (int i = 0; i < docs.length; i++) { // NOPMD - This for loop can be replaced by a foreach loop - MultiGetItemResponse doc = docs[i]; - VXAccessAudit vXAccessAudit = populateViewBean(doc.getResponse()); - if (vXAccessAudit != null) { - String serviceType = vXAccessAudit.getServiceType(); - boolean isHive = "hive".equalsIgnoreCase(serviceType); - if (!hiveQueryVisibility && isHive) { - vXAccessAudit.setRequestData(null); - } else if (isHive) { - String accessType = vXAccessAudit.getAccessType(); - if ("grant".equalsIgnoreCase(accessType) - || "revoke".equalsIgnoreCase(accessType)) { - String requestData = vXAccessAudit.getRequestData(); - if (requestData != null) { - try { - vXAccessAudit.setRequestData( - java.net.URLDecoder.decode(requestData, "UTF-8")); - } catch (UnsupportedEncodingException e) { - LOGGER.warn("Error while encoding request data: " + requestData, e); - } - } else { - LOGGER.warn( - "Error in request data of audit from elasticSearch. AuditData: " - + vXAccessAudit.toString()); - } - } - } - } - xAccessAuditList.add(vXAccessAudit); - } - - VXAccessAuditList returnList = new VXAccessAuditList(); - returnList.setPageSize(searchCriteria.getMaxRows()); - returnList.setResultSize(response.getHits().getHits().length); - returnList.setTotalCount(response.getHits().getTotalHits().value); - returnList.setStartIndex(searchCriteria.getStartIndex()); - returnList.setVXAccessAudits(xAccessAuditList); - return returnList; - } - - public void setRestErrorUtil(RESTErrorUtil restErrorUtil) { - this.restErrorUtil = restErrorUtil; - } - - - /** - * @param doc - * @return - */ - private VXAccessAudit populateViewBean(GetResponse doc) { - VXAccessAudit accessAudit = new VXAccessAudit(); - - Object value = null; - if(LOGGER.isDebugEnabled()) { - LOGGER.debug("doc=" + doc.toString()); - } - - Map<String, Object> source = doc.getSource(); - value = source.get("id"); - if (value != null) { - // TODO: Converting ID to hashcode for now - accessAudit.setId((long) value.hashCode()); - } - - value = source.get("cluster"); - if (value != null) { - accessAudit.setClusterName(value.toString()); - } - - value = source.get("zoneName"); - if (value != null) { - accessAudit.setZoneName(value.toString()); - } - - value = source.get("agentHost"); - if (value != null) { - accessAudit.setAgentHost(value.toString()); - } - - value = source.get("policyVersion"); - if (value != null) { - accessAudit.setPolicyVersion(MiscUtil.toLong(value)); - } - - value = source.get("access"); - if (value != null) { - accessAudit.setAccessType(value.toString()); - } - - value = source.get("enforcer"); - if (value != null) { - accessAudit.setAclEnforcer(value.toString()); - } - value = source.get("agent"); - if (value != null) { - accessAudit.setAgentId(value.toString()); - } - value = source.get("repo"); - if (value != null) { - accessAudit.setRepoName(value.toString()); - XXService xxService = daoManager.getXXService().findByName(accessAudit.getRepoName()); - - if(xxService != null) { - accessAudit.setRepoDisplayName(xxService.getDisplayName()); - } - } - value = source.get("sess"); - if (value != null) { - accessAudit.setSessionId(value.toString()); - } - value = source.get("reqUser"); - if (value != null) { - accessAudit.setRequestUser(value.toString()); - } - value = source.get("reqData"); - if (value != null) { - accessAudit.setRequestData(value.toString()); - } - value = source.get("resource"); - if (value != null) { - accessAudit.setResourcePath(value.toString()); - } - value = source.get("cliIP"); - if (value != null) { - accessAudit.setClientIP(value.toString()); - } - value = source.get("logType"); - //if (value != null) { - // TODO: Need to see what logType maps to in UI -// accessAudit.setAuditType(solrUtil.toInt(value)); - //} - value = source.get("result"); - if (value != null) { - accessAudit.setAccessResult(MiscUtil.toInt(value)); - } - value = source.get("policy"); - if (value != null) { - accessAudit.setPolicyId(MiscUtil.toLong(value)); - } - value = source.get("repoType"); - if (value != null) { - accessAudit.setRepoType(MiscUtil.toInt(value)); - if(null != daoManager) { - XXServiceDefDao xxServiceDef = daoManager.getXXServiceDef(); - if(xxServiceDef != null) { - XXServiceDef xServiceDef = xxServiceDef.getById((long) accessAudit.getRepoType()); - if (xServiceDef != null) { - accessAudit.setServiceType(xServiceDef.getName()); - accessAudit.setServiceTypeDisplayName(xServiceDef.getDisplayName()); - } - } - } - } - value = source.get("resType"); - if (value != null) { - accessAudit.setResourceType(value.toString()); - } - value = source.get("reason"); - if (value != null) { - accessAudit.setResultReason(value.toString()); - } - value = source.get("action"); - if (value != null) { - accessAudit.setAction(value.toString()); - } - value = source.get("evtTime"); - if (value != null) { - accessAudit.setEventTime(MiscUtil.toLocalDate(value)); - } - value = source.get("seq_num"); - if (value != null) { - accessAudit.setSequenceNumber(MiscUtil.toLong(value)); - } - value = source.get("event_count"); - if (value != null) { - accessAudit.setEventCount(MiscUtil.toLong(value)); - } - value = source.get("event_dur_ms"); - if (value != null) { - accessAudit.setEventDuration(MiscUtil.toLong(value)); - } - value = source.get("tags"); - if (value != null) { - accessAudit.setTags(value.toString()); - } - value = source.get("datasets"); - if (value != null) { - try { - accessAudit.setDatasets(JsonUtilsV2.nonSerializableObjToJson(value)); - } catch (Exception e) { - LOGGER.warn("Failed to convert datasets to json", e); - } - } - value = source.get("projects"); - if (value != null) { - try { - accessAudit.setProjects(JsonUtilsV2.nonSerializableObjToJson(value)); - } catch (Exception e) { - LOGGER.warn("Failed to convert projects to json", e); - } - } - return accessAudit; - } - - /** - * @param searchCriteria - * @return - */ - public VXLong getXAccessAuditSearchCount(SearchCriteria searchCriteria) { - long count = 100; - VXLong vXLong = new VXLong(); - vXLong.setValue(count); - return vXLong; - } + private static final Logger LOGGER = LoggerFactory.getLogger(ElasticSearchAccessAuditsService.class); + + @Autowired + ElasticSearchMgr elasticSearchMgr; + + @Autowired + ElasticSearchUtil elasticSearchUtil; + + public VXAccessAuditList searchXAccessAudits(SearchCriteria searchCriteria) { + RestHighLevelClient client = elasticSearchMgr.getClient(); + final boolean hiveQueryVisibility = PropertiesUtil.getBooleanProperty("ranger.audit.hive.query.visibility", true); + + if (client == null) { + LOGGER.warn("ElasticSearch client is null, so not running the query."); + + throw restErrorUtil.createRESTException("Error connecting to search engine", MessageEnums.ERROR_SYSTEM); + } + + List<VXAccessAudit> xAccessAuditList = new ArrayList<VXAccessAudit>(); + Map<String, Object> paramList = searchCriteria.getParamList(); + + updateUserExclusion(paramList); + + SearchResponse response; + + try { + response = elasticSearchUtil.searchResources(searchCriteria, searchFields, sortFields, client, elasticSearchMgr.index); + } catch (IOException e) { + LOGGER.warn("ElasticSearch query failed: {}", e.getMessage()); + + throw restErrorUtil.createRESTException("Error querying search engine", MessageEnums.ERROR_SYSTEM); + } + + MultiGetItemResponse[] docs; + + try { + docs = elasticSearchUtil.fetch(client, elasticSearchMgr.index, response.getHits().getHits()); + } catch (IOException e) { + LOGGER.warn("ElasticSearch fetch failed: {}", e.getMessage()); + + throw restErrorUtil.createRESTException("Error querying search engine", MessageEnums.ERROR_SYSTEM); + } + + for (int i = 0; i < docs.length; i++) { // NOPMD - This for loop can be replaced by a foreach loop + MultiGetItemResponse doc = docs[i]; + VXAccessAudit vXAccessAudit = populateViewBean(doc.getResponse()); + + if (vXAccessAudit != null) { + String serviceType = vXAccessAudit.getServiceType(); + boolean isHive = "hive".equalsIgnoreCase(serviceType); + + if (!hiveQueryVisibility && isHive) { + vXAccessAudit.setRequestData(null); + } else if (isHive) { + String accessType = vXAccessAudit.getAccessType(); + + if ("grant".equalsIgnoreCase(accessType) || "revoke".equalsIgnoreCase(accessType)) { + String requestData = vXAccessAudit.getRequestData(); + + if (requestData != null) { + try { + vXAccessAudit.setRequestData(java.net.URLDecoder.decode(requestData, "UTF-8")); + } catch (UnsupportedEncodingException e) { + LOGGER.warn("Error while encoding request data: {}", requestData, e); + } + } else { + LOGGER.warn("Error in request data of audit from elasticSearch. AuditData: {}", vXAccessAudit); + } + } + } + } + + xAccessAuditList.add(vXAccessAudit); + } + + VXAccessAuditList returnList = new VXAccessAuditList(); + + returnList.setPageSize(searchCriteria.getMaxRows()); + returnList.setResultSize(response.getHits().getHits().length); + returnList.setTotalCount(response.getHits().getTotalHits().value); + returnList.setStartIndex(searchCriteria.getStartIndex()); + returnList.setVXAccessAudits(xAccessAuditList); + + return returnList; + } + + public void setRestErrorUtil(RESTErrorUtil restErrorUtil) { + this.restErrorUtil = restErrorUtil; + } + + /** + * @param searchCriteria + * @return + */ + public VXLong getXAccessAuditSearchCount(SearchCriteria searchCriteria) { + long count = 100; + VXLong vXLong = new VXLong(); + + vXLong.setValue(count); + + return vXLong; + } + + /** + * @param doc + * @return + */ + private VXAccessAudit populateViewBean(GetResponse doc) { + LOGGER.debug("doc={}", doc); + + VXAccessAudit accessAudit = new VXAccessAudit(); + Map<String, Object> source = doc.getSource(); + Object value; + + value = source.get("id"); + if (value != null) { + // TODO: Converting ID to hashcode for now + accessAudit.setId((long) value.hashCode()); + } + + value = source.get("cluster"); + if (value != null) { + accessAudit.setClusterName(value.toString()); + } + + value = source.get("zoneName"); + if (value != null) { + accessAudit.setZoneName(value.toString()); + } + + value = source.get("agentHost"); + if (value != null) { + accessAudit.setAgentHost(value.toString()); + } + + value = source.get("policyVersion"); + if (value != null) { + accessAudit.setPolicyVersion(MiscUtil.toLong(value)); + } + + value = source.get("access"); + if (value != null) { + accessAudit.setAccessType(value.toString()); + } + + value = source.get("enforcer"); + if (value != null) { + accessAudit.setAclEnforcer(value.toString()); + } + + value = source.get("agent"); + if (value != null) { + accessAudit.setAgentId(value.toString()); + } + + value = source.get("repo"); + if (value != null) { + accessAudit.setRepoName(value.toString()); + + XXService xxService = daoManager.getXXService().findByName(accessAudit.getRepoName()); + + if (xxService != null) { + accessAudit.setRepoDisplayName(xxService.getDisplayName()); + } + } + + value = source.get("sess"); + if (value != null) { + accessAudit.setSessionId(value.toString()); + } + + value = source.get("reqUser"); + if (value != null) { + accessAudit.setRequestUser(value.toString()); + } + + value = source.get("reqData"); + if (value != null) { + accessAudit.setRequestData(value.toString()); + } + + value = source.get("resource"); + if (value != null) { + accessAudit.setResourcePath(value.toString()); + } + + value = source.get("cliIP"); + if (value != null) { + accessAudit.setClientIP(value.toString()); + } + + // TODO: Need to see what logType maps to in UI + //value = source.get("logType"); + //if (value != null) { + // accessAudit.setAuditType(solrUtil.toInt(value)); + //} + + value = source.get("result"); + if (value != null) { + accessAudit.setAccessResult(MiscUtil.toInt(value)); + } + + value = source.get("policy"); + if (value != null) { + accessAudit.setPolicyId(MiscUtil.toLong(value)); + } + + value = source.get("repoType"); + if (value != null) { + accessAudit.setRepoType(MiscUtil.toInt(value)); + + if (null != daoManager) { + XXServiceDefDao xxServiceDef = daoManager.getXXServiceDef(); + + if (xxServiceDef != null) { + XXServiceDef xServiceDef = xxServiceDef.getById((long) accessAudit.getRepoType()); + + if (xServiceDef != null) { + accessAudit.setServiceType(xServiceDef.getName()); + accessAudit.setServiceTypeDisplayName(xServiceDef.getDisplayName()); + } + } + } + } + + value = source.get("resType"); + if (value != null) { + accessAudit.setResourceType(value.toString()); + } + + value = source.get("reason"); + if (value != null) { + accessAudit.setResultReason(value.toString()); + } + + value = source.get("action"); + if (value != null) { + accessAudit.setAction(value.toString()); + } + + value = source.get("evtTime"); + if (value != null) { + accessAudit.setEventTime(MiscUtil.toLocalDate(value)); + } + + value = source.get("seq_num"); + if (value != null) { + accessAudit.setSequenceNumber(MiscUtil.toLong(value)); + } + + value = source.get("event_count"); + if (value != null) { + accessAudit.setEventCount(MiscUtil.toLong(value)); + } + + value = source.get("event_dur_ms"); + if (value != null) { + accessAudit.setEventDuration(MiscUtil.toLong(value)); + } + + value = source.get("tags"); + if (value != null) { + accessAudit.setTags(value.toString()); + } + + value = source.get("datasets"); + if (value != null) { + try { + accessAudit.setDatasets(JsonUtilsV2.nonSerializableObjToJson(value)); + } catch (Exception e) { + LOGGER.warn("Failed to convert datasets to json", e); + } + } + + value = source.get("projects"); + if (value != null) { + try { + accessAudit.setProjects(JsonUtilsV2.nonSerializableObjToJson(value)); + } catch (Exception e) { + LOGGER.warn("Failed to convert projects to json", e); + } + } + return accessAudit; + } } diff --git a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchMgr.java b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchMgr.java index 6987f6eb1..289578290 100644 --- a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchMgr.java @@ -47,110 +47,134 @@ import java.security.PrivilegedActionException; import java.util.Date; import java.util.Locale; -import static org.apache.ranger.audit.destination.ElasticSearchAuditDestination.*; +import static org.apache.ranger.audit.destination.ElasticSearchAuditDestination.CONFIG_INDEX; +import static org.apache.ranger.audit.destination.ElasticSearchAuditDestination.CONFIG_PORT; +import static org.apache.ranger.audit.destination.ElasticSearchAuditDestination.CONFIG_PREFIX; +import static org.apache.ranger.audit.destination.ElasticSearchAuditDestination.CONFIG_PROTOCOL; +import static org.apache.ranger.audit.destination.ElasticSearchAuditDestination.CONFIG_PWRD; +import static org.apache.ranger.audit.destination.ElasticSearchAuditDestination.CONFIG_URLS; +import static org.apache.ranger.audit.destination.ElasticSearchAuditDestination.CONFIG_USER; /** * This class initializes the ElasticSearch client - * */ @Component public class ElasticSearchMgr { + private static final Logger logger = LoggerFactory.getLogger(ElasticSearchMgr.class); + + public String index; + + Subject subject; + String user; + String password; + RestHighLevelClient client; + + public static RestClientBuilder getRestClientBuilder(String urls, String protocol, String user, String password, int port) { + RestClientBuilder restClientBuilder = RestClient.builder(MiscUtil.toArray(urls, ",").stream().map(x -> new HttpHost(x, port, protocol)).<HttpHost>toArray(i -> new HttpHost[i])); + + if (StringUtils.isNotBlank(user) && StringUtils.isNotBlank(password) && !user.equalsIgnoreCase("NONE") && !password.equalsIgnoreCase("NONE")) { + if (password.contains("keytab") && new File(password).exists()) { + final KerberosCredentialsProvider credentialsProvider = CredentialsProviderUtil.getKerberosCredentials(user, password); + Lookup<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create().register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory()).build(); + + restClientBuilder.setHttpClientConfigCallback(clientBuilder -> { + clientBuilder.setDefaultCredentialsProvider(credentialsProvider); + clientBuilder.setDefaultAuthSchemeRegistry(authSchemeRegistry); + + return clientBuilder; + }); + } else { + final CredentialsProvider credentialsProvider = CredentialsProviderUtil.getBasicCredentials(user, password); + + restClientBuilder.setHttpClientConfigCallback(clientBuilder -> clientBuilder.setDefaultCredentialsProvider(credentialsProvider)); + } + } else { + logger.error("ElasticSearch Credentials not provided!!"); + + final CredentialsProvider credentialsProvider = null; + + restClientBuilder.setHttpClientConfigCallback(clientBuilder -> clientBuilder.setDefaultCredentialsProvider(credentialsProvider)); + } + + return restClientBuilder; + } + + public RestHighLevelClient getClient() { + RestHighLevelClient me = client; + + if (me != null && subject != null) { + KerberosTicket ticket = CredentialsProviderUtil.getTGT(subject); + + try { + if (new Date().getTime() > ticket.getEndTime().getTime()) { + client = null; + CredentialsProviderUtil.ticketExpireTime80 = 0; + + me = connect(); + } else if (CredentialsProviderUtil.ticketWillExpire(ticket)) { + subject = CredentialsProviderUtil.login(user, password); + } + } catch (PrivilegedActionException e) { + logger.error("PrivilegedActionException:", e); + + throw new RuntimeException(e); + } + + return me; + } else { + me = connect(); + } + + return me; + } + + synchronized RestHighLevelClient connect() { + RestHighLevelClient me = client; + + if (me == null) { + synchronized (ElasticSearchAuditDestination.class) { + me = client; + + if (me == null) { + String urls = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_URLS); + String protocol = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_PROTOCOL, "http"); + + user = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_USER, ""); + password = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_PWRD, ""); + + int port = Integer.parseInt(PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_PORT)); + + this.index = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_INDEX, "ranger_audits"); + + String parameterString = String.format(Locale.ROOT, "User:%s, %s://%s:%s/%s", user, protocol, urls, port, index); + + logger.info("Initializing ElasticSearch {}", parameterString); + + if (urls != null) { + urls = urls.trim(); + } + + if (StringUtils.isBlank(urls) || "NONE".equalsIgnoreCase(urls.trim())) { + logger.info("Clearing URI config value: {}", urls); + + urls = null; + } + + try { + if (StringUtils.isNotBlank(user) && StringUtils.isNotBlank(password) && password.contains("keytab") && new File(password).exists()) { + subject = CredentialsProviderUtil.login(user, password); + } + + RestClientBuilder restClientBuilder = getRestClientBuilder(urls, protocol, user, password, port); - private static final Logger logger = LoggerFactory.getLogger(ElasticSearchMgr.class); - public String index; - Subject subject; - String user; - String password; - - synchronized void connect() { - if (client == null) { - synchronized (ElasticSearchAuditDestination.class) { - if (client == null) { - - String urls = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_URLS); - String protocol = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_PROTOCOL, "http"); - user = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_USER, ""); - password = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_PWRD, ""); - int port = Integer.parseInt(PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_PORT)); - this.index = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_INDEX, "ranger_audits"); - String parameterString = String.format(Locale.ROOT,"User:%s, %s://%s:%s/%s", user, protocol, urls, port, index); - logger.info("Initializing ElasticSearch " + parameterString); - if (urls != null) { - urls = urls.trim(); - } - if (StringUtils.isBlank(urls) || "NONE".equalsIgnoreCase(urls.trim())) { - logger.info(String.format("Clearing URI config value: %s", urls)); - urls = null; - } - - try { - if (StringUtils.isNotBlank(user) && StringUtils.isNotBlank(password) && password.contains("keytab") && new File(password).exists()) { - subject = CredentialsProviderUtil.login(user, password); - } - RestClientBuilder restClientBuilder = - getRestClientBuilder(urls, protocol, user, password, port); - client = new RestHighLevelClient(restClientBuilder); - } catch (Throwable t) { - logger.error("Can't connect to ElasticSearch: " + parameterString, t); - } - } - } - } - } - - public static RestClientBuilder getRestClientBuilder(String urls, String protocol, String user, String password, int port) { - RestClientBuilder restClientBuilder = RestClient.builder( - MiscUtil.toArray(urls, ",").stream() - .map(x -> new HttpHost(x, port, protocol)) - .<HttpHost>toArray(i -> new HttpHost[i]) - ); - if (StringUtils.isNotBlank(user) && StringUtils.isNotBlank(password) && !user.equalsIgnoreCase("NONE") && !password.equalsIgnoreCase("NONE")) { - if (password.contains("keytab") && new File(password).exists()) { - final KerberosCredentialsProvider credentialsProvider = - CredentialsProviderUtil.getKerberosCredentials(user, password); - Lookup<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create() - .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory()).build(); - restClientBuilder.setHttpClientConfigCallback(clientBuilder -> { - clientBuilder.setDefaultCredentialsProvider(credentialsProvider); - clientBuilder.setDefaultAuthSchemeRegistry(authSchemeRegistry); - return clientBuilder; - }); - } else { - final CredentialsProvider credentialsProvider = - CredentialsProviderUtil.getBasicCredentials(user, password); - restClientBuilder.setHttpClientConfigCallback(clientBuilder -> - clientBuilder.setDefaultCredentialsProvider(credentialsProvider)); - } - } else { - logger.error("ElasticSearch Credentials not provided!!"); - final CredentialsProvider credentialsProvider = null; - restClientBuilder.setHttpClientConfigCallback(clientBuilder -> - clientBuilder.setDefaultCredentialsProvider(credentialsProvider)); - } - return restClientBuilder; - } - - RestHighLevelClient client = null; - public RestHighLevelClient getClient() { - if (client != null && subject != null) { - KerberosTicket ticket = CredentialsProviderUtil.getTGT(subject); - try { - if (new Date().getTime() > ticket.getEndTime().getTime()){ - client = null; - CredentialsProviderUtil.ticketExpireTime80 = 0; - connect(); - } else if (CredentialsProviderUtil.ticketWillExpire(ticket)) { - subject = CredentialsProviderUtil.login(user, password); - } - } catch (PrivilegedActionException e) { - logger.error("PrivilegedActionException:", e); - throw new RuntimeException(e); - } - return client; - } else { - connect(); - } - return client; - } + client = new RestHighLevelClient(restClientBuilder); + } catch (Throwable t) { + logger.error("Can't connect to ElasticSearch: {}", parameterString, t); + } + } + } + } + return me; + } } diff --git a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchUtil.java b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchUtil.java index 134436667..f9cbcbe43 100644 --- a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchUtil.java @@ -19,7 +19,11 @@ package org.apache.ranger.elasticsearch; -import org.apache.ranger.common.*; +import org.apache.ranger.common.PropertiesUtil; +import org.apache.ranger.common.SearchCriteria; +import org.apache.ranger.common.SearchField; +import org.apache.ranger.common.SortField; +import org.apache.ranger.common.StringUtil; import org.apache.solr.client.solrj.util.ClientUtils; import org.elasticsearch.action.get.MultiGetItemResponse; import org.elasticsearch.action.get.MultiGetRequest; @@ -42,10 +46,11 @@ import org.springframework.stereotype.Component; import java.io.IOException; import java.text.SimpleDateFormat; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.List; +import java.util.TimeZone; @Component public class ElasticSearchUtil { @@ -54,17 +59,19 @@ public class ElasticSearchUtil { @Autowired StringUtil stringUtil; - String dateFormateStr = "yyyy-MM-dd'T'HH:mm:ss'Z'"; - SimpleDateFormat dateFormat = new SimpleDateFormat(dateFormateStr); + final String dateFormateStr = "yyyy-MM-dd'T'HH:mm:ss'Z'"; + final SimpleDateFormat dateFormat = new SimpleDateFormat(dateFormateStr); public ElasticSearchUtil() { String timeZone = PropertiesUtil.getProperty("xa.elasticSearch.timezone"); + if (timeZone != null) { - logger.info("Setting timezone to " + timeZone); + logger.info("Setting timezone to {}", timeZone); + try { dateFormat.setTimeZone(TimeZone.getTimeZone(timeZone)); } catch (Throwable t) { - logger.error("Error setting timezone. TimeZone = " + timeZone); + logger.error("Error setting timezone. TimeZone = {}", timeZone); } } } @@ -72,8 +79,10 @@ public class ElasticSearchUtil { public SearchResponse searchResources(SearchCriteria searchCriteria, List<SearchField> searchFields, List<SortField> sortFields, RestHighLevelClient client, String index) throws IOException { // See Also: https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high-query-builders.html QueryAccumulator queryAccumulator = new QueryAccumulator(searchCriteria); + if (searchCriteria.getParamList() != null) { searchFields.stream().forEach(queryAccumulator::addQuery); + // For now assuming there is only date field where range query will // be done. If we there are more than one, then we should create a // hashmap for each field name @@ -81,31 +90,39 @@ public class ElasticSearchUtil { queryAccumulator.queries.add(setDateRange(queryAccumulator.dateFieldName, queryAccumulator.fromDate, queryAccumulator.toDate)); } } + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + queryAccumulator.queries.stream().filter(x -> x != null).forEach(boolQueryBuilder::must); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + setSortClause(searchCriteria, sortFields, searchSourceBuilder); + searchSourceBuilder.from(searchCriteria.getStartIndex()); searchSourceBuilder.size(searchCriteria.getMaxRows()); searchSourceBuilder.fetchSource(true); + SearchRequest query = new SearchRequest(); + query.indices(index); query.source(searchSourceBuilder.query(boolQueryBuilder)); + return client.search(query, RequestOptions.DEFAULT); } - public void setSortClause(SearchCriteria searchCriteria, - List<SortField> sortFields, - SearchSourceBuilder searchSourceBuilder) { - + public void setSortClause(SearchCriteria searchCriteria, List<SortField> sortFields, SearchSourceBuilder searchSourceBuilder) { // TODO: We are supporting single sort field only for now - String sortBy = searchCriteria.getSortBy(); + String sortBy = searchCriteria.getSortBy(); String querySortBy = null; + if (!stringUtil.isEmpty(sortBy)) { sortBy = sortBy.trim(); + for (SortField sortField : sortFields) { if (sortBy.equalsIgnoreCase(sortField.getParamName())) { querySortBy = sortField.getFieldName(); + // Override the sortBy using the normalized value searchCriteria.setSortBy(sortField.getParamName()); break; @@ -117,6 +134,7 @@ public class ElasticSearchUtil { for (SortField sortField : sortFields) { if (sortField.isDefault()) { querySortBy = sortField.getFieldName(); + // Override the sortBy using the default value searchCriteria.setSortBy(sortField.getParamName()); searchCriteria.setSortType(sortField.getDefaultOrder().name()); @@ -127,11 +145,13 @@ public class ElasticSearchUtil { if (querySortBy != null) { // Add sort type - String sortType = searchCriteria.getSortType(); - SortOrder order = SortOrder.ASC; - if (sortType != null && "desc".equalsIgnoreCase(sortType)) { + String sortType = searchCriteria.getSortType(); + SortOrder order = SortOrder.ASC; + + if ("desc".equalsIgnoreCase(sortType)) { order = SortOrder.DESC; } + searchSourceBuilder.sort(querySortBy, order); } } @@ -140,6 +160,7 @@ public class ElasticSearchUtil { if (valueList == null || valueList.isEmpty()) { return null; } + if (valueList.isEmpty()) { return null; } else { @@ -152,63 +173,73 @@ public class ElasticSearchUtil { } } - - private String filterText(Object value) { - return ClientUtils.escapeQueryChars(value.toString().trim().toLowerCase()); - } - public QueryBuilder setDateRange(String fieldName, Date fromDate, Date toDate) { RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery(fieldName).format(dateFormateStr); + if (fromDate != null) { rangeQueryBuilder.from(dateFormat.format(fromDate)); } + if (toDate != null) { rangeQueryBuilder.to(dateFormat.format(toDate)); } + return rangeQueryBuilder; } public MultiGetItemResponse[] fetch(RestHighLevelClient client, String index, SearchHit... hits) throws IOException { - if(0 == hits.length) { + if (0 == hits.length) { return new MultiGetItemResponse[0]; } + MultiGetRequest multiGetRequest = new MultiGetRequest(); + for (SearchHit hit : hits) { MultiGetRequest.Item item = new MultiGetRequest.Item(index, null, hit.getId()); + item.fetchSourceContext(FetchSourceContext.FETCH_SOURCE); + multiGetRequest.add(item); } + return client.multiGet(multiGetRequest, RequestOptions.DEFAULT).getResponses(); } + private String filterText(Object value) { + return ClientUtils.escapeQueryChars(value.toString().trim().toLowerCase()); + } + private class QueryAccumulator { public final List<QueryBuilder> queries = new ArrayList<>(); - public final SearchCriteria searchCriteria; - public Date fromDate; - public Date toDate; - public String dateFieldName; + public final SearchCriteria searchCriteria; + public Date fromDate; + public Date toDate; + public String dateFieldName; private QueryAccumulator(SearchCriteria searchCriteria) { this.searchCriteria = searchCriteria; - this.fromDate = null; - this.toDate = null; - this.dateFieldName = null; + this.fromDate = null; + this.toDate = null; + this.dateFieldName = null; } public QueryAccumulator addQuery(SearchField searchField) { QueryBuilder queryBuilder = getQueryBuilder(searchField); + if (null != queryBuilder) { queries.add(queryBuilder); } + return this; } public QueryBuilder getQueryBuilder(SearchField searchField) { - String clientFieldName = searchField.getClientFieldName(); - String fieldName = searchField.getFieldName(); - SearchField.DATA_TYPE dataType = searchField.getDataType(); - SearchField.SEARCH_TYPE searchType = searchField.getSearchType(); - Object paramValue = searchCriteria.getParamValue(clientFieldName); + String clientFieldName = searchField.getClientFieldName(); + String fieldName = searchField.getFieldName(); + SearchField.DATA_TYPE dataType = searchField.getDataType(); + SearchField.SEARCH_TYPE searchType = searchField.getSearchType(); + Object paramValue = searchCriteria.getParamValue(clientFieldName); + return getQueryBuilder(dataType, searchType, fieldName, paramValue); } @@ -216,12 +247,15 @@ public class ElasticSearchUtil { if (paramValue == null || paramValue.toString().isEmpty()) { return null; } + if (fieldName.startsWith("-")) { QueryBuilder negativeQuery = getQueryBuilder(dataType, searchType, fieldName.substring(1), paramValue); return null == negativeQuery ? null : QueryBuilders.boolQuery().mustNot(negativeQuery); } + if (paramValue instanceof Collection) { Collection<?> valueList = (Collection<?>) paramValue; + if (valueList.isEmpty()) { return null; } else { @@ -235,27 +269,22 @@ public class ElasticSearchUtil { } else { if (dataType == SearchField.DATA_TYPE.DATE) { if (!(paramValue instanceof Date)) { - logger.error(String.format( - "Search value is not a Java Date Object: %s %s %s", - fieldName, searchType, paramValue)); + logger.error("Search value is not a Java Date Object: {} {} {}", fieldName, searchType, paramValue); } else { - if (searchType == SearchField.SEARCH_TYPE.GREATER_EQUAL_THAN - || searchType == SearchField.SEARCH_TYPE.GREATER_THAN) { - fromDate = (Date) paramValue; + if (searchType == SearchField.SEARCH_TYPE.GREATER_EQUAL_THAN || searchType == SearchField.SEARCH_TYPE.GREATER_THAN) { + fromDate = (Date) paramValue; dateFieldName = fieldName; - } else if (searchType == SearchField.SEARCH_TYPE.LESS_EQUAL_THAN - || searchType == SearchField.SEARCH_TYPE.LESS_THAN) { - toDate = (Date) paramValue; + } else if (searchType == SearchField.SEARCH_TYPE.LESS_EQUAL_THAN || searchType == SearchField.SEARCH_TYPE.LESS_THAN) { + toDate = (Date) paramValue; dateFieldName = fieldName; } } + return null; - } else if (searchType == SearchField.SEARCH_TYPE.GREATER_EQUAL_THAN - || searchType == SearchField.SEARCH_TYPE.GREATER_THAN - || searchType == SearchField.SEARCH_TYPE.LESS_EQUAL_THAN - || searchType == SearchField.SEARCH_TYPE.LESS_THAN) { //NOPMD - logger.warn(String.format("Range Queries Not Implemented: %s %s %s", - fieldName, searchType, paramValue)); + } else if (searchType == SearchField.SEARCH_TYPE.GREATER_EQUAL_THAN || searchType == SearchField.SEARCH_TYPE.GREATER_THAN + || searchType == SearchField.SEARCH_TYPE.LESS_EQUAL_THAN || searchType == SearchField.SEARCH_TYPE.LESS_THAN) { //NOPMD + logger.warn("Range Queries Not Implemented: {} {} {}", fieldName, searchType, paramValue); + return null; } else { if (searchType == SearchField.SEARCH_TYPE.PARTIAL) { @@ -274,6 +303,5 @@ public class ElasticSearchUtil { } } } - } }
