This is an automated email from the ASF dual-hosted git repository. ramackri pushed a commit to branch backport-audit-pr3-plugins-2.10 in repository https://gitbox.apache.org/repos/asf/ranger.git
commit 6aa5873fb039d8f32220222a85e27712c445d383 Author: ramk <[email protected]> AuthorDate: Mon Sep 28 23:56:14 2026 +0530 RANGER-5482: Plugin audit-server destination for ranger-2.10 (PR-3) Add ranger-audit-dest-auditserver and wire plugins/agents to post audits to PR-1 ingestor; jersey-bundle packaging fixes for KMS/Knox without root pom or master-only reformat churn. --- .../dest-auditserver}/pom.xml | 42 +-- .../destination/RangerAuditServerDestination.java | 383 +++++++++++++++++++++ agents-audit/pom.xml | 1 + agents-common/pom.xml | 7 +- agents-common/scripts/enable-agent.sh | 5 +- .../plugin/util/NashornScriptEngineCreator.java | 31 +- authz-embedded/pom.xml | 8 +- distro/pom.xml | 6 + distro/src/main/assembly/hbase-agent.xml | 71 ++-- distro/src/main/assembly/hdfs-agent.xml | 6 +- distro/src/main/assembly/hive-agent.xml | 38 +- distro/src/main/assembly/kms.xml | 3 +- distro/src/main/assembly/knox-agent.xml | 72 ++-- distro/src/main/assembly/pdp.xml | 3 +- distro/src/main/assembly/plugin-atlas.xml | 3 +- distro/src/main/assembly/plugin-elasticsearch.xml | 3 +- distro/src/main/assembly/plugin-kafka.xml | 74 ++-- distro/src/main/assembly/plugin-kms.xml | 18 +- distro/src/main/assembly/plugin-kylin.xml | 3 +- distro/src/main/assembly/plugin-ozone.xml | 61 ++-- distro/src/main/assembly/plugin-presto.xml | 3 +- distro/src/main/assembly/plugin-solr.xml | 11 +- distro/src/main/assembly/plugin-sqoop.xml | 3 +- distro/src/main/assembly/plugin-trino.xml | 3 +- distro/src/main/assembly/plugin-yarn.xml | 56 +-- distro/src/main/assembly/storm-agent.xml | 3 +- hbase-agent/conf/ranger-hbase-audit-changes.cfg | 5 + hbase-agent/pom.xml | 28 +- hbase-agent/scripts/install.properties | 4 + hdfs-agent/conf/ranger-hdfs-audit-changes.cfg | 5 + hdfs-agent/conf/ranger-hdfs-audit.xml | 19 +- hdfs-agent/pom.xml | 12 +- hdfs-agent/scripts/install.properties | 4 + hive-agent/conf/ranger-hive-audit-changes.cfg | 5 + hive-agent/conf/ranger-hive-audit.xml | 19 +- hive-agent/pom.xml | 7 +- hive-agent/scripts/install.properties | 4 + kms/pom.xml | 15 - kms/scripts/install.properties | 4 + knox-agent/conf/ranger-knox-audit-changes.cfg | 5 + knox-agent/pom.xml | 36 +- knox-agent/scripts/install.properties | 4 + plugin-atlas/conf/ranger-atlas-audit-changes.cfg | 5 + plugin-atlas/pom.xml | 7 +- plugin-atlas/scripts/install.properties | 4 + .../conf/ranger-elasticsearch-audit-changes.cfg | 5 + plugin-elasticsearch/pom.xml | 7 +- plugin-elasticsearch/scripts/install.properties | 4 + plugin-kafka/conf/ranger-kafka-audit-changes.cfg | 5 + plugin-kafka/pom.xml | 7 +- plugin-kafka/scripts/install.properties | 4 + plugin-kafka/template/configuration.xml | 20 ++ plugin-kms/conf/ranger-kms-audit-changes.cfg | 5 + plugin-kms/pom.xml | 17 +- plugin-kudu/pom.xml | 7 +- plugin-kylin/conf/ranger-kylin-audit-changes.cfg | 5 + plugin-kylin/pom.xml | 17 +- plugin-kylin/scripts/install.properties | 4 + plugin-nifi-registry/pom.xml | 7 +- plugin-nifi/pom.xml | 7 +- plugin-ozone/conf/ranger-ozone-audit-changes.cfg | 5 + plugin-ozone/pom.xml | 37 +- plugin-ozone/scripts/install.properties | 4 + plugin-presto/conf/ranger-presto-audit-changes.cfg | 5 + plugin-presto/pom.xml | 7 +- plugin-presto/scripts/install.properties | 4 + plugin-solr/conf/ranger-solr-audit-changes.cfg | 5 + plugin-solr/pom.xml | 7 +- plugin-solr/scripts/install.properties | 4 + plugin-sqoop/conf/ranger-sqoop-audit-changes.cfg | 5 + plugin-sqoop/pom.xml | 7 +- plugin-sqoop/scripts/install.properties | 4 + plugin-trino/conf/ranger-trino-audit-changes.cfg | 5 + plugin-trino/pom.xml | 7 +- plugin-trino/scripts/install.properties | 4 + plugin-yarn/conf/ranger-yarn-audit-changes.cfg | 5 + plugin-yarn/pom.xml | 7 +- plugin-yarn/scripts/install.properties | 4 + plugin-yarn/template/configuration.xml | 20 ++ ranger-examples/plugin-sampleapp/pom.xml | 7 +- ranger-kylin-plugin-shim/pom.xml | 10 + storm-agent/conf/ranger-storm-audit-changes.cfg | 5 + storm-agent/scripts/install.properties | 4 + 83 files changed, 973 insertions(+), 429 deletions(-) diff --git a/plugin-kudu/pom.xml b/agents-audit/dest-auditserver/pom.xml similarity index 65% copy from plugin-kudu/pom.xml copy to agents-audit/dest-auditserver/pom.xml index 555a94a4d..8ae78d940 100644 --- a/plugin-kudu/pom.xml +++ b/agents-audit/dest-auditserver/pom.xml @@ -21,53 +21,49 @@ <groupId>org.apache.ranger</groupId> <artifactId>ranger</artifactId> <version>2.10.0-SNAPSHOT</version> - <relativePath>..</relativePath> + <relativePath>../..</relativePath> </parent> - <artifactId>ranger-kudu-plugin</artifactId> + <artifactId>ranger-audit-dest-auditserver</artifactId> <packaging>jar</packaging> - <name>Kudu Security Plugin</name> - <description>Kudu Security Plugin</description> + <name>Ranger Audit Destination - auditserver</name> + <description>Ranger Audit Destination - auditserver</description> <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <securesm.version>1.2</securesm.version> </properties> <dependencies> <dependency> - <groupId>org.apache.ranger</groupId> - <artifactId>credentialbuilder</artifactId> - <version>${project.version}</version> + <groupId>com.fasterxml.jackson.jaxrs</groupId> + <artifactId>jackson-jaxrs-json-provider</artifactId> + <version>${fasterxml.jackson.version}</version> </dependency> <dependency> <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-hdfs</artifactId> + <artifactId>ranger-audit-core</artifactId> <version>${project.version}</version> </dependency> <dependency> <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-solr</artifactId> + <artifactId>ranger-plugins-common</artifactId> <version>${project.version}</version> </dependency> <dependency> - <groupId>org.apache.ranger</groupId> - <artifactId>ranger-plugins-common</artifactId> - <version>${project.version}</version> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + <version>${slf4j.version}</version> </dependency> <!-- Test --> <dependency> - <groupId>org.junit.jupiter</groupId> - <artifactId>junit-jupiter-api</artifactId> - <version>${junit.jupiter.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.junit.vintage</groupId> - <artifactId>junit-vintage-engine</artifactId> - <version>${junit.jupiter.version}</version> + <groupId>org.slf4j</groupId> + <artifactId>log4j-over-slf4j</artifactId> + <version>${slf4j.version}</version> <scope>test</scope> </dependency> <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-core</artifactId> + <groupId>org.testng</groupId> + <artifactId>testng</artifactId> + <version>${testng.version}</version> <scope>test</scope> </dependency> </dependencies> diff --git a/agents-audit/dest-auditserver/src/main/java/org/apache/ranger/audit/destination/RangerAuditServerDestination.java b/agents-audit/dest-auditserver/src/main/java/org/apache/ranger/audit/destination/RangerAuditServerDestination.java new file mode 100644 index 000000000..0dbb01ede --- /dev/null +++ b/agents-audit/dest-auditserver/src/main/java/org/apache/ranger/audit/destination/RangerAuditServerDestination.java @@ -0,0 +1,383 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.destination; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.ranger.audit.model.AuditEventBase; +import org.apache.ranger.audit.model.AuthzAuditEvent; +import org.apache.ranger.audit.provider.MiscUtil; +import org.apache.ranger.plugin.util.RangerRESTClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Properties; + +public class RangerAuditServerDestination extends AuditDestination { + private static final Logger LOG = LoggerFactory.getLogger(RangerAuditServerDestination.class); + + private static final ObjectMapper AUDIT_POST_MAPPER = new ObjectMapper(); + + public static final String PROP_URL = "url"; + public static final String PROP_SSL_CONFIG_FILE = "ssl.config.file"; + public static final String PROP_AUTHN_TYPE = "authn.type"; + public static final String PROP_AUTHN_BASIC_USERNAME = "authn.basic.username"; + public static final String PROP_AUTHN_BASIC_PASSWORD = "authn.basic.password"; + public static final String PROP_AUTHN_JWT_ENV = "authn.jwt.env"; + public static final String PROP_AUTHN_JWT_FILE = "authn.jwt.file"; + public static final String PROP_CLIENT_CONN_TIMEOUT_MS = "connection.timeout.ms"; + public static final String PROP_CLIENT_READ_TIMEOUT_MS = "read.timeout.ms"; + public static final String PROP_MAX_RETRY_ATTEMPTS = "max.retry.attempts"; + public static final String PROP_RETRY_INTERVAL_MS = "retry.interval.ms"; + + public static final String REST_RELATIVE_PATH_POST = "/api/audit/access"; + public static final String QUERY_PARAM_SERVICE_NAME = "serviceName"; + public static final String QUERY_PARAM_APP_ID = "appId"; + + // Authentication types + public static final String AUTH_TYPE_KERBEROS = "kerberos"; + public static final String AUTH_TYPE_BASIC = "basic"; + public static final String AUTH_TYPE_JWT = "jwt"; + + private RangerRESTClient restClient; + + @Override + public void init(Properties props, String propPrefix) { + LOG.info("==> RangerAuditServerDestination:init()"); + + super.init(props, propPrefix); + + String url = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_URL); + String sslConfigFileName = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_SSL_CONFIG_FILE); + int connTimeoutMs = MiscUtil.getIntProperty(props, propPrefix + "." + PROP_CLIENT_CONN_TIMEOUT_MS, 120000); + int readTimeoutMs = MiscUtil.getIntProperty(props, propPrefix + "." + PROP_CLIENT_READ_TIMEOUT_MS, 30000); + int maxRetryAttempts = MiscUtil.getIntProperty(props, propPrefix + "." + PROP_MAX_RETRY_ATTEMPTS, 3); + int retryIntervalMs = MiscUtil.getIntProperty(props, propPrefix + "." + PROP_RETRY_INTERVAL_MS, 1000); + String authType = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_AUTHN_TYPE); + + LOG.info("Audit destination authentication type: {}", authType); + + Configuration config = createRESTClientConfiguration(props, propPrefix, authType); + + this.restClient = new RangerRESTClient(url, sslConfigFileName, config); + + this.restClient.setRestClientConnTimeOutMs(connTimeoutMs); + this.restClient.setRestClientReadTimeOutMs(readTimeoutMs); + this.restClient.setMaxRetryAttempts(maxRetryAttempts); + this.restClient.setRetryIntervalMs(retryIntervalMs); + + LOG.info("<== RangerAuditServerDestination:init()"); + } + + @Override + public void stop() { + LOG.info("==> RangerAuditServerDestination.stop() called.."); + + logStatus(); + + RangerRESTClient restClient = this.restClient; + + if (restClient != null) { + restClient.resetClient(); + + this.restClient = null; + } + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#flush() + */ + @Override + public void flush() { + } + + @Override + public boolean log(Collection<AuditEventBase> events) { + boolean ret = false; + + try { + logStatusIfRequired(); + + addTotalCount(events.size()); + + RangerRESTClient restClient = this.restClient; + + if (restClient != null) { + ret = logAsBatch(events, restClient); + } else { + LOG.error("REST client is not initialized. Cannot send audit events"); + + addDeferredCount(events.size()); + } + } catch (Throwable t) { + logError("Error sending audit to Audit Server", t); + + addDeferredCount(events.size()); + } + + return ret; + } + + private boolean logAsBatch(Collection<AuditEventBase> events, RangerRESTClient restClient) { + int totalEvents = events.size(); + + LOG.debug("==> logAsBatch(): sending batch of {} events to Audit Server", totalEvents); + + boolean batchSuccess = sendBatch(events, restClient); + + if (batchSuccess) { + addSuccessCount(totalEvents); + } else { + LOG.error("Failed to send batch of {} events", totalEvents); + + addFailedCount(totalEvents); + } + + LOG.debug("<== logAsBatch(): batch processing complete: {}/{} events sent successfully", batchSuccess ? totalEvents : 0, totalEvents); + + return batchSuccess; + } + + private boolean sendBatch(Collection<AuditEventBase> events, RangerRESTClient restClient) { + boolean ret; + Map<String, String> queryParams = new HashMap<>(); + String serviceName = fetchServiceName(events); + String appId = fetchAppId(events); + + LOG.debug("Adding serviceName={} to audit request", serviceName); + + queryParams.put(QUERY_PARAM_SERVICE_NAME, serviceName); + + if (StringUtils.isNotEmpty(appId)) { + LOG.debug("Adding appId={} to audit request for batch processing", appId); + + queryParams.put(QUERY_PARAM_APP_ID, appId); + } + + try { + final UserGroupInformation user = MiscUtil.getUGILoginUser(); + final boolean isSecureMode = isKerberosAuthenticated(); + + if (isSecureMode && user != null) { + LOG.debug("Sending audit batch of {} events using Kerberos. Principal: {}, AuthMethod: {}", events.size(), user.getUserName(), user.getAuthenticationMethod()); + } else { + LOG.debug("Sending audit batch of {} events. SecureMode: {}, User: {}", events.size(), isSecureMode, user != null ? user.getUserName() : "null"); + } + + final ClientResponse response; + + if (isSecureMode) { + response = MiscUtil.executePrivilegedAction((PrivilegedExceptionAction<ClientResponse>) () -> { + try { + return postAuditEvents(restClient, queryParams, events); + } catch (Exception e) { + LOG.error("Failed to post audit events in privileged action: {}", e.getMessage()); + throw e; + } + }); + } else { + response = postAuditEvents(restClient, queryParams, events); + } + + if (response != null) { + int status = response.getStatus(); + + if (status == 200) { + if (LOG.isDebugEnabled()) { + LOG.debug("Audit batch sent successfully. {} events delivered. Response: {}", events.size(), response.getEntity(String.class)); + } + + ret = true; + } else { + String errorBody = ""; + + try { + if (response.hasEntity()) { + errorBody = response.getEntity(String.class); + } + } catch (Exception e) { + LOG.debug("Failed to read error response body", e); + } + + LOG.error("Failed to send audit batch. HTTP status: {}, Response: {}", status, errorBody); + + if (status == 401) { + LOG.error("Authentication failure (401). Verify credentials are valid and audit server is properly configured."); + } + + ret = false; + } + } else { + LOG.error("Received null response from audit server for batch of {} events", events.size()); + + ret = false; + } + } catch (Exception e) { + LOG.error("Failed to send audit batch of {} events. Error: {}", events.size(), e.getMessage(), e); + + ret = false; + } + + return ret; + } + + private ClientResponse postAuditEvents(RangerRESTClient restClient, Map<String, String> params, Collection<AuditEventBase> events) { + LOG.debug("Posting {} audit events to {}", events.size(), REST_RELATIVE_PATH_POST); + + WebResource webResource = restClient.getResource(REST_RELATIVE_PATH_POST); + + if (params != null && !params.isEmpty()) { + for (Map.Entry<String, String> entry : params.entrySet()) { + webResource = webResource.queryParam(entry.getKey(), entry.getValue()); + } + } + + // Pre-serialize with ISO-8601 dates: Jersey 1.x on plugin classloader threads cannot + // reliably resolve List body writers, and JsonUtils uses a non-ingestor date format. + String jsonBody; + + try { + jsonBody = AUDIT_POST_MAPPER.writeValueAsString(new ArrayList<>(events)); + } catch (Exception e) { + throw new RuntimeException("Failed to serialize audit events to JSON", e); + } + + return webResource + .accept("application/json") + .type("application/json") + .entity(jsonBody) + .post(ClientResponse.class); + } + + private static Configuration createRESTClientConfiguration(Properties props, String propPrefix, String authType) { + Configuration config = new Configuration(); + + for (String key : props.stringPropertyNames()) { + config.set(key, props.getProperty(key)); + } + + final String restClientPrefix = "ranger.plugin"; + + if (AUTH_TYPE_JWT.equalsIgnoreCase(authType)) { + String jwtEnv = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_AUTHN_JWT_ENV); + String jwtFile = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_AUTHN_JWT_FILE); + + if (StringUtils.isNotBlank(jwtEnv)) { + config.set(restClientPrefix + ".policy.rest.client.jwt.source", "env"); + config.set(restClientPrefix + ".policy.rest.client.jwt.env", jwtEnv); + } else if (StringUtils.isNotBlank(jwtFile)) { + config.set(restClientPrefix + ".policy.rest.client.jwt.source", "file"); + config.set(restClientPrefix + ".policy.rest.client.jwt.file", jwtFile); + } else { + LOG.warn("JWT authentication configured but no token found. Configure {} or {}", PROP_AUTHN_JWT_ENV, PROP_AUTHN_JWT_FILE); + } + } else if (AUTH_TYPE_KERBEROS.equalsIgnoreCase(authType)) { + initKerberos(); + + // For Kerberos, no additional configuration is needed in RangerRESTClient + // Authentication happens via Subject.doAs() security context + LOG.info("Kerberos authentication will be used via privileged action (Subject.doAs)"); + } else if (AUTH_TYPE_BASIC.equalsIgnoreCase(authType)) { + String userName = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_AUTHN_BASIC_USERNAME); + String password = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_AUTHN_BASIC_PASSWORD); + + if (StringUtils.isNotBlank(userName) && StringUtils.isNotBlank(password)) { + config.set(restClientPrefix + ".policy.rest.client.username", userName); + config.set(restClientPrefix + ".policy.rest.client.password", password); + + LOG.info("Basic authentication configured for user: {}", userName); + } else { + LOG.warn("Basic authentication configured but username/password not found. Configure {} and {}", PROP_AUTHN_BASIC_USERNAME, PROP_AUTHN_BASIC_PASSWORD); + } + } + + return config; + } + + private static void initKerberos() { + LOG.info("==> RangerAuditServerDestination.initKerberos()"); + + try { + UserGroupInformation ugi = UserGroupInformation.getLoginUser(); + + if (ugi == null) { + LOG.warn("No UserGroupInformation available for Kerberos authentication"); + } else if (!ugi.hasKerberosCredentials()) { + LOG.warn("User {} does not have Kerberos credentials for authentication", ugi.getUserName()); + } else { + LOG.info("Kerberos authentication for user: {}, authMethod: {}", ugi.getUserName(), ugi.getAuthenticationMethod()); + + ugi.checkTGTAndReloginFromKeytab(); + + LOG.debug("TGT verified and refreshed if needed for user: {}", ugi.getUserName()); + + LOG.info("Kerberos authentication completed successfully"); + } + } catch (Exception e) { + LOG.warn("Kerberos authentication failed. First request will retry authentication", e); + } + + LOG.info("<== RangerAuditServerDestination.initKerberos()"); + } + + private static boolean isKerberosAuthenticated() { + try { + boolean isSecurityEnabled = UserGroupInformation.isSecurityEnabled(); + + if (isSecurityEnabled) { + UserGroupInformation loggedInUser = UserGroupInformation.getLoginUser(); + + return loggedInUser != null && + loggedInUser.hasKerberosCredentials() && + UserGroupInformation.AuthenticationMethod.KERBEROS.equals(loggedInUser.getAuthenticationMethod()); + } + } catch (Exception e) { + LOG.warn("Failed to check Kerberos authentication status", e); + } + + return false; + } + + private static String fetchServiceName(Collection<AuditEventBase> events) { + Iterator<AuditEventBase> iter = events.iterator(); + AuditEventBase auditEvent = iter.hasNext() ? iter.next() : null; + + return (auditEvent instanceof AuthzAuditEvent) ? ((AuthzAuditEvent) auditEvent).getRepositoryName() : null; + } + + private static String fetchAppId(Collection<AuditEventBase> events) { + Iterator<AuditEventBase> iter = events.iterator(); + AuditEventBase auditEvent = iter.hasNext() ? iter.next() : null; + + return (auditEvent instanceof AuthzAuditEvent) ? ((AuthzAuditEvent) auditEvent).getAgentId() : null; + } +} diff --git a/agents-audit/pom.xml b/agents-audit/pom.xml index 5ca02ca31..193e97ac2 100644 --- a/agents-audit/pom.xml +++ b/agents-audit/pom.xml @@ -31,6 +31,7 @@ <!-- Sub modules --> <modules> <module>core</module> + <module>dest-auditserver</module> <module>dest-cloudwatch</module> <module>dest-es</module> <module>dest-hdfs</module> diff --git a/agents-common/pom.xml b/agents-common/pom.xml index 5b2508b9b..3cf4f126a 100644 --- a/agents-common/pom.xml +++ b/agents-common/pom.xml @@ -69,12 +69,7 @@ </dependency> <dependency> <groupId>com.sun.jersey</groupId> - <artifactId>jersey-client</artifactId> - <version>${jersey-bundle.version}</version> - </dependency> - <dependency> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-core</artifactId> + <artifactId>jersey-bundle</artifactId> <version>${jersey-bundle.version}</version> </dependency> <dependency> diff --git a/agents-common/scripts/enable-agent.sh b/agents-common/scripts/enable-agent.sh index d4c7816b5..6899bb1be 100755 --- a/agents-common/scripts/enable-agent.sh +++ b/agents-common/scripts/enable-agent.sh @@ -205,9 +205,10 @@ elif [ "${HCOMPONENT_NAME}" = "storm" ]; then HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/extlib-daemon elif [ "${HCOMPONENT_NAME}" = "atlas" ]; then HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/libext -elif [ "${HCOMPONENT_NAME}" = "hadoop" ] || - [ "${HCOMPONENT_NAME}" = "yarn" ]; then +elif [ "${HCOMPONENT_NAME}" = "hadoop" ]; then HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/share/hadoop/hdfs/lib +elif [ "${HCOMPONENT_NAME}" = "yarn" ]; then + HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/share/hadoop/yarn/lib elif [ "${HCOMPONENT_NAME}" = "sqoop" ]; then HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/server/lib elif [ "${HCOMPONENT_NAME}" = "kylin" ]; then diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/NashornScriptEngineCreator.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/NashornScriptEngineCreator.java index 4af3d5c63..8e9f1ae2e 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/NashornScriptEngineCreator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/NashornScriptEngineCreator.java @@ -23,14 +23,18 @@ import org.slf4j.LoggerFactory; import javax.script.ScriptEngine; -import jdk.nashorn.api.scripting.ClassFilter; -import jdk.nashorn.api.scripting.NashornScriptEngineFactory; + +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; public class NashornScriptEngineCreator implements ScriptEngineCreator { private static final Logger LOG = LoggerFactory.getLogger(NashornScriptEngineCreator.class); private static final String[] SCRIPT_ENGINE_ARGS = new String[] { "--no-java", "--no-syntax-extensions" }; private static final String ENGINE_NAME = "NashornScriptEngine"; + private static final String NASHORN_FACTORY = "jdk.nashorn.api.scripting.NashornScriptEngineFactory"; + private static final String CLASS_FILTER = "jdk.nashorn.api.scripting.ClassFilter"; @Override public ScriptEngine getScriptEngine(ClassLoader clsLoader) { @@ -41,9 +45,13 @@ public ScriptEngine getScriptEngine(ClassLoader clsLoader) { } try { - NashornScriptEngineFactory factory = new NashornScriptEngineFactory(); + Class<?> classFilterType = Class.forName(CLASS_FILTER, true, clsLoader); + Class<?> factoryClass = Class.forName(NASHORN_FACTORY, true, clsLoader); + Object classFilter = Proxy.newProxyInstance(clsLoader, new Class<?>[] { classFilterType }, RangerClassFilterHandler.INSTANCE); + Object factory = factoryClass.getDeclaredConstructor().newInstance(); + Method getScriptEngine = factoryClass.getMethod("getScriptEngine", String[].class, ClassLoader.class, classFilterType); - ret = factory.getScriptEngine(SCRIPT_ENGINE_ARGS, clsLoader, RangerClassFilter.INSTANCE); + ret = (ScriptEngine) getScriptEngine.invoke(factory, SCRIPT_ENGINE_ARGS, clsLoader, classFilter); } catch (Throwable t) { LOG.debug("NashornScriptEngineCreator.getScriptEngine(): failed to create engine type {}", ENGINE_NAME, t); } @@ -51,17 +59,20 @@ public ScriptEngine getScriptEngine(ClassLoader clsLoader) { return ret; } - private static class RangerClassFilter implements ClassFilter { - static final RangerClassFilter INSTANCE = new RangerClassFilter(); + private static final class RangerClassFilterHandler implements InvocationHandler { + static final RangerClassFilterHandler INSTANCE = new RangerClassFilterHandler(); - private RangerClassFilter() { + private RangerClassFilterHandler() { } @Override - public boolean exposeToScripts(String className) { - LOG.warn("script blocked: attempt to use Java class {}", className); + public Object invoke(Object proxy, Method method, Object[] args) { + if ("exposeToScripts".equals(method.getName()) && args != null && args.length == 1 && args[0] instanceof String) { + LOG.warn("script blocked: attempt to use Java class {}", args[0]); + return Boolean.FALSE; + } - return false; + return null; } } } diff --git a/authz-embedded/pom.xml b/authz-embedded/pom.xml index 08bf4e018..259098cdd 100644 --- a/authz-embedded/pom.xml +++ b/authz-embedded/pom.xml @@ -41,13 +41,7 @@ <dependency> <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-hdfs</artifactId> - <version>${project.version}</version> - </dependency> - - <dependency> - <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-solr</artifactId> + <artifactId>ranger-audit-dest-auditserver</artifactId> <version>${project.version}</version> </dependency> diff --git a/distro/pom.xml b/distro/pom.xml index 8b9a72104..3986b217a 100644 --- a/distro/pom.xml +++ b/distro/pom.xml @@ -83,6 +83,12 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> + <dependency> + <groupId>org.apache.ranger</groupId> + <artifactId>ranger-audit-dest-auditserver</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> <dependency> <groupId>org.apache.ranger</groupId> <artifactId>ranger-audit-dest-cloudwatch</artifactId> diff --git a/distro/src/main/assembly/hbase-agent.xml b/distro/src/main/assembly/hbase-agent.xml index 204d7e135..ff867d5ca 100644 --- a/distro/src/main/assembly/hbase-agent.xml +++ b/distro/src/main/assembly/hbase-agent.xml @@ -42,8 +42,7 @@ <useAllReactorProjects>true</useAllReactorProjects> <includes> <include>org.apache.ranger:ranger-audit-core</include> - <include>org.apache.ranger:ranger-audit-dest-hdfs</include> - <include>org.apache.ranger:ranger-audit-dest-solr</include> + <include>org.apache.ranger:ranger-audit-dest-auditserver</include> <include>org.apache.ranger:ranger-authz-api</include> <include>org.apache.ranger:ranger-common-utils</include> <include>org.apache.ranger:ranger-plugins-cred</include> @@ -58,35 +57,50 @@ <directoryMode>755</directoryMode> <fileMode>644</fileMode> <includes> - <include>commons-collections:commons-collections</include> - <include>com.sun.jersey:jersey-client:jar:${jersey-bundle.version}</include> - <include>com.sun.jersey:jersey-core:jar:${jersey-bundle.version}</include> - <include>com.fasterxml.jackson.core:jackson-annotations:jar:${fasterxml.jackson.version}</include> - <include>com.fasterxml.jackson.core:jackson-core:jar:${fasterxml.jackson.version}</include> - <include>com.fasterxml.jackson.core:jackson-databind:jar:${fasterxml.jackson.version}</include> - <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar:${fasterxml.jackson.version}</include> - <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar:${fasterxml.jackson.version}</include> - <include>org.eclipse.jetty:jetty-client:jar:${jetty-client.version}</include> - <include>org.apache.httpcomponents:httpmime:jar:${httpcomponents.httpmime.version}</include> - <include>org.noggit:noggit:jar:${noggit.version}</include> - <include>org.apache.solr:solr-solrj:jar:${solr.version}</include> - <include>org.elasticsearch:elasticsearch</include> - <include>org.elasticsearch:elasticsearch-core</include> - <include>org.elasticsearch:elasticsearch-x-content</include> - <include>org.elasticsearch.client:elasticsearch-rest-client</include> - <include>org.elasticsearch.client:elasticsearch-rest-high-level-client</include> - <include>org.elasticsearch.plugin:rank-eval-client</include> - <include>org.elasticsearch.plugin:lang-mustache-client</include> - <include>org.apache.httpcomponents:httpasyncclient:jar:${httpcomponents.httpasyncclient.version}</include> - <include>org.apache.lucene:lucene-core</include> - <include>joda-time:joda-time</include> <include>com.carrotsearch:hppc</include> - <include>org.apache.hive:hive-storage-api:jar:${hive.storage-api.version}</include> - <include>org.apache.orc:orc-core:jar:${orc.version}</include> - <include>org.apache.orc:orc-shims:jar:${orc.version}</include> + <include>com.fasterxml.woodstox:woodstox-core</include> + <include>com.google.code.gson:gson</include> + <include>com.sun.jersey:jersey-bundle:jar:${jersey-bundle.version}</include> + <include>com.fasterxml.jackson.core:jackson-core</include> + <include>com.fasterxml.jackson.core:jackson-databind</include> + <include>com.fasterxml.jackson.core:jackson-annotations</include> + <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base</include> + <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider</include> + <include>com.fasterxml.jackson.module:jackson-module-jaxb-annotations</include> + <include>commons-codec:commons-codec</include> + <include>commons-collections:commons-collections</include> + <include>commons-io:commons-io</include> + <include>commons-logging:commons-logging</include> <include>io.airlift:aircompressor:jar:${aircompressor.version}</include> + <include>joda-time:joda-time</include> + <include>org.apache.commons:commons-collections4:jar:${commons.collections4.version}</include> + <include>org.apache.commons:commons-compress:jar:${commons.compress.version}</include> + <include>org.apache.commons:commons-configuration2:jar:${commons.configuration.version}</include> <include>org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version}</include> + <include>org.apache.httpcomponents:httpasyncclient:jar:${httpcomponents.httpasyncclient.version}</include> + <include>org.apache.httpcomponents:httpclient:jar:${httpcomponents.httpclient.version}</include> + <include>org.apache.httpcomponents:httpcore:jar:${httpcomponents.httpcore.version}</include> + <include>org.apache.httpcomponents:httpmime:jar:${httpcomponents.httpmime.version}</include> + <include>org.apache.solr:solr-solrj:jar:${solr.version}</include> + <include>org.codehaus.woodstox:stax2-api</include> + <include>org.eclipse.jetty:jetty-client:jar:${jetty-client.version}</include> + <include>org.graalvm.js:js:jar:${graalvm.version}</include> + <include>org.graalvm.js:js-scriptengine:jar:${graalvm.version}</include> + <include>org.graalvm.regex:regex:jar:${graalvm.version}</include> + <include>org.graalvm.sdk:graal-sdk:jar:${graalvm.version}</include> + <include>org.graalvm.truffle:truffle-api:jar:${graalvm.version}</include> + <include>com.ibm.icu:icu4j</include> </includes> + <excludes> + <exclude>com.fasterxml.jackson.dataformat:*</exclude> + <exclude>org.glassfish.jersey.core:*</exclude> + <exclude>org.glassfish.jersey.ext:*</exclude> + <exclude>org.glassfish.jersey.inject:*</exclude> + <exclude>org.glassfish.jersey.media:*</exclude> + <exclude>org.glassfish.hk2:*</exclude> + <exclude>javax.ws.rs:javax.ws.rs-api</exclude> + <exclude>jakarta.ws.rs:jakarta.ws.rs-api</exclude> + </excludes> </binaries> </moduleSet> @@ -105,15 +119,14 @@ <include>commons-cli:commons-cli</include> <include>commons-collections:commons-collections</include> <include>commons-io:commons-io:jar:${commons.io.version}</include> - <include>commons-logging:commons-logging:jar:${commons.logging.version}</include> <include>org.apache.commons:commons-collections4:jar:${commons.collections4.version}</include> <include>org.apache.commons:commons-compress:jar:${commons.compress.version}</include> <include>org.apache.commons:commons-configuration2:jar:${commons.configuration.version}</include> <include>org.apache.commons:commons-lang3:jar:${commons.lang3.version}</include> + <include>org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version}</include> <include>org.apache.hadoop:hadoop-auth:jar:${hadoop.version}</include> <include>org.apache.hadoop:hadoop-client-api:jar:${hadoop.version}</include> <include>org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version}</include> - <include>org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version}</include> <include>org.codehaus.woodstox:stax2-api</include> <include>org.slf4j:slf4j-api:jar:${slf4j-api.version}</include> </includes> diff --git a/distro/src/main/assembly/hdfs-agent.xml b/distro/src/main/assembly/hdfs-agent.xml index 46c68b9d5..ee006ff8c 100644 --- a/distro/src/main/assembly/hdfs-agent.xml +++ b/distro/src/main/assembly/hdfs-agent.xml @@ -72,8 +72,7 @@ <useAllReactorProjects>true</useAllReactorProjects> <includes> <include>org.apache.ranger:ranger-audit-core</include> - <include>org.apache.ranger:ranger-audit-dest-hdfs</include> - <include>org.apache.ranger:ranger-audit-dest-solr</include> + <include>org.apache.ranger:ranger-audit-dest-auditserver</include> <include>org.apache.ranger:ranger-authz-api</include> <include>org.apache.ranger:ranger-common-utils</include> <include>org.apache.ranger:ranger-plugins-cred</include> @@ -114,6 +113,9 @@ <include>org.apache.orc:orc-shims:jar:${orc.version}</include> <include>io.airlift:aircompressor:jar:${aircompressor.version}</include> <include>org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version}</include> + <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar:${fasterxml.jackson.version}</include> + <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar:${fasterxml.jackson.version}</include> + <include>com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar:${fasterxml.jackson.version}</include> </includes> </binaries> </moduleSet> diff --git a/distro/src/main/assembly/hive-agent.xml b/distro/src/main/assembly/hive-agent.xml index bef0a220c..c26ea72af 100644 --- a/distro/src/main/assembly/hive-agent.xml +++ b/distro/src/main/assembly/hive-agent.xml @@ -42,8 +42,7 @@ <useAllReactorProjects>true</useAllReactorProjects> <includes> <include>org.apache.ranger:ranger-audit-core</include> - <include>org.apache.ranger:ranger-audit-dest-hdfs</include> - <include>org.apache.ranger:ranger-audit-dest-solr</include> + <include>org.apache.ranger:ranger-audit-dest-auditserver</include> <include>org.apache.ranger:ranger-authz-api</include> <include>org.apache.ranger:ranger-common-utils</include> <include>org.apache.ranger:ranger-plugins-cred</include> @@ -58,26 +57,22 @@ <directoryMode>755</directoryMode> <fileMode>644</fileMode> <includes> - <include>commons-collections:commons-collections</include> - <include>org.eclipse.jetty:jetty-client:jar:${jetty-client.version}</include> + <include>jakarta.ws.rs:jakarta.ws.rs-api</include> + <include>org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version}</include> + <include>org.apache.httpcomponents:httpasyncclient:jar:${httpcomponents.httpasyncclient.version}</include> <include>org.apache.httpcomponents:httpmime:jar:${httpcomponents.httpmime.version}</include> - <include>org.apache.httpcomponents:httpclient:jar:${httpcomponents.httpclient.version}</include> - <include>org.apache.httpcomponents:httpcore:jar:${httpcomponents.httpcore.version}</include> - <include>org.noggit:noggit:jar:${noggit.version}</include> <include>org.apache.solr:solr-solrj:jar:${solr.version}</include> - <include>org.elasticsearch:elasticsearch</include> - <include>org.elasticsearch:elasticsearch-core</include> - <include>org.elasticsearch:elasticsearch-x-content</include> - <include>org.elasticsearch.client:elasticsearch-rest-client</include> - <include>org.elasticsearch.client:elasticsearch-rest-high-level-client</include> - <include>org.elasticsearch.plugin:rank-eval-client</include> - <include>org.elasticsearch.plugin:lang-mustache-client</include> - <include>org.apache.httpcomponents:httpcore-nio:jar:${httpcomponents.httpcore.version}</include> - <include>org.apache.httpcomponents:httpasyncclient:jar:${httpcomponents.httpasyncclient.version}</include> - <include>org.apache.lucene:lucene-core</include> - <include>joda-time:joda-time</include> - <include>com.carrotsearch:hppc</include> - <include>org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version}</include> + <include>org.eclipse.jetty:jetty-client:jar:${jetty-client.version}</include> + <include>org.glassfish.jersey.core:jersey-client</include> + <include>org.glassfish.jersey.core:jersey-common</include> + <include>org.glassfish.jersey.media:jersey-media-json-jackson</include> + <include>com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar:2.16.1</include> + <include>org.graalvm.js:js:jar:${graalvm.version}</include> + <include>org.graalvm.js:js-scriptengine:jar:${graalvm.version}</include> + <include>org.graalvm.regex:regex:jar:${graalvm.version}</include> + <include>org.graalvm.sdk:graal-sdk:jar:${graalvm.version}</include> + <include>org.graalvm.truffle:truffle-api:jar:${graalvm.version}</include> + <include>com.ibm.icu:icu4j</include> </includes> </binaries> </moduleSet> @@ -97,15 +92,14 @@ <include>commons-cli:commons-cli</include> <include>commons-collections:commons-collections</include> <include>commons-io:commons-io:jar:${commons.io.version}</include> - <include>commons-logging:commons-logging:jar:${commons.logging.version}</include> <include>org.apache.commons:commons-collections4:jar:${commons.collections4.version}</include> <include>org.apache.commons:commons-compress:jar:${commons.compress.version}</include> <include>org.apache.commons:commons-configuration2:jar:${commons.configuration.version}</include> <include>org.apache.commons:commons-lang3:jar:${commons.lang3.version}</include> + <include>org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version}</include> <include>org.apache.hadoop:hadoop-auth:jar:${hadoop.version}</include> <include>org.apache.hadoop:hadoop-client-api:jar:${hadoop.version}</include> <include>org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version}</include> - <include>org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version}</include> <include>org.codehaus.woodstox:stax2-api</include> <include>org.slf4j:slf4j-api:jar:${slf4j-api.version}</include> </includes> diff --git a/distro/src/main/assembly/kms.xml b/distro/src/main/assembly/kms.xml index c96d1e7f4..4587e6d82 100755 --- a/distro/src/main/assembly/kms.xml +++ b/distro/src/main/assembly/kms.xml @@ -281,8 +281,7 @@ <useAllReactorProjects>true</useAllReactorProjects> <includes> <include>org.apache.ranger:ranger-audit-core</include> - <include>org.apache.ranger:ranger-audit-dest-hdfs</include> - <include>org.apache.ranger:ranger-audit-dest-solr</include> + <include>org.apache.ranger:ranger-audit-dest-auditserver</include> <include>org.apache.ranger:ranger-authz-api</include> <include>org.apache.ranger:ranger-common-utils</include> <include>org.apache.ranger:ranger-plugins-cred</include> diff --git a/distro/src/main/assembly/knox-agent.xml b/distro/src/main/assembly/knox-agent.xml index 72eaec4c9..f0465f2a2 100644 --- a/distro/src/main/assembly/knox-agent.xml +++ b/distro/src/main/assembly/knox-agent.xml @@ -43,8 +43,7 @@ <useAllReactorProjects>true</useAllReactorProjects> <includes> <include>org.apache.ranger:ranger-audit-core</include> - <include>org.apache.ranger:ranger-audit-dest-hdfs</include> - <include>org.apache.ranger:ranger-audit-dest-solr</include> + <include>org.apache.ranger:ranger-audit-dest-auditserver</include> <include>org.apache.ranger:ranger-authz-api</include> <include>org.apache.ranger:ranger-common-utils</include> <include>org.apache.ranger:ranger-plugins-cred</include> @@ -59,40 +58,52 @@ <directoryMode>755</directoryMode> <fileMode>644</fileMode> <includes> + <include>org.apache.ranger:ranger-audit-core</include> + <include>org.apache.ranger:ranger-audit-dest-auditserver</include> + <include>com.carrotsearch:hppc</include> + <include>com.fasterxml.jackson.core:jackson-annotations:jar:${fasterxml.jackson.version}</include> + <include>com.fasterxml.jackson.core:jackson-core:jar:${fasterxml.jackson.version}</include> + <include>com.fasterxml.jackson.core:jackson-databind:jar:${fasterxml.jackson.version}</include> + <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar:${fasterxml.jackson.version}</include> + <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar:${fasterxml.jackson.version}</include> + <include>com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar:${fasterxml.jackson.version}</include> + <include>com.fasterxml.woodstox:woodstox-core</include> + <include>com.google.code.gson:gson*</include> + <include>com.google.protobuf:protobuf-java:jar:${protobuf-java.version}</include> + <include>commons-codec:commons-codec</include> <include>commons-collections:commons-collections</include> + <include>commons-io:commons-io</include> + <include>commons-logging:commons-logging</include> + <include>io.airlift:aircompressor:jar:${aircompressor.version}</include> + <include>javax.inject:javax.inject</include> <include>com.sun.jersey:jersey-client:jar:${jersey-bundle.version}</include> + <include>com.sun.jersey:jersey-core:jar:${jersey-bundle.version}</include> + <include>joda-time:joda-time</include> <include>org.apache.commons:commons-configuration2</include> - <include>com.google.code.gson:gson*</include> - <include>org.apache.httpcomponents:httpmime:jar:${httpcomponents.httpmime.version}</include> - <include>org.apache.httpcomponents:httpclient:jar:${httpcomponents.httpclient.version}</include> - <include>org.apache.httpcomponents:httpcore:jar:${httpcomponents.httpcore.version}</include> - <include>org.noggit:noggit:jar:${noggit.version}</include> - <include>com.google.protobuf:protobuf-java:jar:${protobuf-java.version}</include> + <include>org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version}</include> <include>org.apache.hadoop:hadoop-client-api:jar:${hadoop.version}</include> <include>org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version}</include> - <include>org.apache.hadoop:hadoop-hdfs-client:jar:${hadoop.version}</include> - <include>com.fasterxml.jackson.core:jackson-annotations:jar:${fasterxml.jackson.version}</include> - <include>com.fasterxml.jackson.core:jackson-core:jar:${fasterxml.jackson.version}</include> - <include>com.fasterxml.jackson.core:jackson-databind:jar:${fasterxml.jackson.version}</include> - <include>org.apache.solr:solr-solrj:jar:${solr.version}</include> - <include>org.elasticsearch:elasticsearch</include> - <include>org.elasticsearch:elasticsearch-core</include> - <include>org.elasticsearch:elasticsearch-x-content</include> - <include>org.elasticsearch.client:elasticsearch-rest-client</include> - <include>org.elasticsearch.client:elasticsearch-rest-high-level-client</include> - <include>org.elasticsearch.plugin:rank-eval-client</include> - <include>org.elasticsearch.plugin:lang-mustache-client</include> - <include>org.apache.httpcomponents:httpcore-nio:jar:${httpcomponents.httpcore.version}</include> <include>org.apache.httpcomponents:httpasyncclient:jar:${httpcomponents.httpasyncclient.version}</include> - <include>org.apache.lucene:lucene-core</include> - <include>joda-time:joda-time</include> - <include>com.carrotsearch:hppc</include> - <include>org.apache.hive:hive-storage-api:jar:${hive.storage-api.version}</include> - <include>org.apache.orc:orc-core:jar:${orc.version}</include> - <include>org.apache.orc:orc-shims:jar:${orc.version}</include> - <include>io.airlift:aircompressor:jar:${aircompressor.version}</include> - <include>org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version}</include> + <include>org.apache.httpcomponents:httpclient:jar:${httpcomponents.httpclient.version}</include> + <include>org.apache.httpcomponents:httpcore-nio:jar:${httpcomponents.httpcore.version}</include> + <include>org.apache.httpcomponents:httpcore:jar:${httpcomponents.httpcore.version}</include> + <include>org.apache.httpcomponents:httpmime:jar:${httpcomponents.httpmime.version}</include> + <include>org.apache.solr:solr-solrj:jar:${solr.version}</include> + <include>org.codehaus.woodstox:stax2-api</include> + <include>org.eclipse.jetty:jetty-client:jar:${jetty-client.version}</include> </includes> + <excludes> + <exclude>com.fasterxml.jackson.dataformat:*</exclude> + <exclude>com.sun.jersey:jersey-bundle</exclude> + <exclude>org.glassfish.jersey.core:*</exclude> + <exclude>org.glassfish.jersey.ext:*</exclude> + <exclude>org.glassfish.jersey.inject:*</exclude> + <exclude>org.glassfish.jersey.media:*</exclude> + <exclude>org.glassfish.hk2:*</exclude> + <exclude>javax.ws.rs:javax.ws.rs-api</exclude> + <exclude>javax.ws.rs:jsr311-api</exclude> + <exclude>jakarta.ws.rs:jakarta.ws.rs-api</exclude> + </excludes> </binaries> </moduleSet> @@ -111,15 +122,14 @@ <include>commons-cli:commons-cli</include> <include>commons-collections:commons-collections</include> <include>commons-io:commons-io:jar:${commons.io.version}</include> - <include>commons-logging:commons-logging:jar:${commons.logging.version}</include> <include>org.apache.commons:commons-collections4:jar:${commons.collections4.version}</include> <include>org.apache.commons:commons-compress:jar:${commons.compress.version}</include> <include>org.apache.commons:commons-configuration2:jar:${commons.configuration.version}</include> <include>org.apache.commons:commons-lang3:jar:${commons.lang3.version}</include> + <include>org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version}</include> <include>org.apache.hadoop:hadoop-auth:jar:${hadoop.version}</include> <include>org.apache.hadoop:hadoop-client-api:jar:${hadoop.version}</include> <include>org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version}</include> - <include>org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version}</include> <include>org.codehaus.woodstox:stax2-api</include> <include>org.slf4j:slf4j-api:jar:${slf4j-api.version}</include> </includes> diff --git a/distro/src/main/assembly/pdp.xml b/distro/src/main/assembly/pdp.xml index e2cd5f807..a9f1c9b2d 100644 --- a/distro/src/main/assembly/pdp.xml +++ b/distro/src/main/assembly/pdp.xml @@ -32,8 +32,7 @@ <include>org.apache.ranger:credentialbuilder</include> <include>org.apache.ranger:authz-embedded</include> <include>org.apache.ranger:ranger-audit-core</include> - <include>org.apache.ranger:ranger-audit-dest-hdfs</include> - <include>org.apache.ranger:ranger-audit-dest-solr</include> + <include>org.apache.ranger:ranger-audit-dest-auditserver</include> <include>org.apache.ranger:ranger-authn</include> <include>org.apache.ranger:ranger-authz-api</include> <include>org.apache.ranger:ranger-common-utils</include> diff --git a/distro/src/main/assembly/plugin-atlas.xml b/distro/src/main/assembly/plugin-atlas.xml index 166c49991..56123eb3b 100644 --- a/distro/src/main/assembly/plugin-atlas.xml +++ b/distro/src/main/assembly/plugin-atlas.xml @@ -43,8 +43,7 @@ <useAllReactorProjects>true</useAllReactorProjects> <includes> <include>org.apache.ranger:ranger-audit-core</include> - <include>org.apache.ranger:ranger-audit-dest-hdfs</include> - <include>org.apache.ranger:ranger-audit-dest-solr</include> + <include>org.apache.ranger:ranger-audit-dest-auditserver</include> <include>org.apache.ranger:ranger-authz-api</include> <include>org.apache.ranger:ranger-common-utils</include> <include>org.apache.ranger:ranger-plugins-cred</include> diff --git a/distro/src/main/assembly/plugin-elasticsearch.xml b/distro/src/main/assembly/plugin-elasticsearch.xml index 0f29303c5..3efe53b7a 100644 --- a/distro/src/main/assembly/plugin-elasticsearch.xml +++ b/distro/src/main/assembly/plugin-elasticsearch.xml @@ -48,8 +48,7 @@ <useAllReactorProjects>true</useAllReactorProjects> <includes> <include>org.apache.ranger:ranger-audit-core</include> - <include>org.apache.ranger:ranger-audit-dest-es</include> - <include>org.apache.ranger:ranger-audit-dest-hdfs</include> + <include>org.apache.ranger:ranger-audit-dest-auditserver</include> <include>org.apache.ranger:ranger-authz-api</include> <include>org.apache.ranger:ranger-common-utils</include> <include>org.apache.ranger:ranger-plugins-cred</include> diff --git a/distro/src/main/assembly/plugin-kafka.xml b/distro/src/main/assembly/plugin-kafka.xml index 655f43b74..39fa437e7 100644 --- a/distro/src/main/assembly/plugin-kafka.xml +++ b/distro/src/main/assembly/plugin-kafka.xml @@ -38,8 +38,7 @@ <useAllReactorProjects>true</useAllReactorProjects> <includes> <include>org.apache.ranger:ranger-audit-core</include> - <include>org.apache.ranger:ranger-audit-dest-hdfs</include> - <include>org.apache.ranger:ranger-audit-dest-solr</include> + <include>org.apache.ranger:ranger-audit-dest-auditserver</include> <include>org.apache.ranger:ranger-authz-api</include> <include>org.apache.ranger:ranger-common-utils</include> <include>org.apache.ranger:ranger-kafka-plugin</include> @@ -54,53 +53,53 @@ <directoryMode>755</directoryMode> <fileMode>644</fileMode> <includes> + <include>com.carrotsearch:hppc</include> + <include>com.fasterxml.woodstox:woodstox-core</include> + <include>com.google.code.gson:gson</include> + <include>com.sun.jersey:jersey-bundle:jar:${jersey-bundle.version}</include> + <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base</include> + <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider</include> + <include>com.fasterxml.jackson.module:jackson-module-jaxb-annotations</include> + <include>com.google.protobuf:protobuf-java:jar:${protobuf-java.version}</include> + <include>commons-codec:commons-codec</include> + <include>commons-collections:commons-collections</include> + <include>commons-io:commons-io</include> + <include>commons-logging:commons-logging</include> + <include>io.airlift:aircompressor:jar:${aircompressor.version}</include> + <include>joda-time:joda-time</include> <include>org.apache.commons:commons-collections4:jar:${commons.collections4.version}</include> <include>org.apache.commons:commons-compress:jar:${commons.compress.version}</include> <include>org.apache.commons:commons-configuration2:jar:${commons.configuration.version}</include> + <include>org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version}</include> <include>org.apache.hadoop:hadoop-auth:jar:${hadoop.version}</include> <include>org.apache.hadoop:hadoop-client-api:jar:${hadoop.version}</include> <include>org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version}</include> - <include>org.apache.hadoop:hadoop-hdfs-client:jar:${hadoop.version}</include> - <include>com.google.code.gson:gson</include> - <include>org.eclipse.jetty:jetty-client:jar:${jetty-client.version}</include> - <include>commons-collections:commons-collections</include> - <include>com.sun.jersey:jersey-client:jar:${jersey-bundle.version}</include> - <include>com.sun.jersey:jersey-core:jar:${jersey-bundle.version}</include> - <include>commons-logging:commons-logging</include> - <include>commons-io:commons-io</include> + <include>org.apache.httpcomponents:httpasyncclient:jar:${httpcomponents.httpasyncclient.version}</include> <include>org.apache.httpcomponents:httpclient:jar:${httpcomponents.httpclient.version}</include> + <include>org.apache.httpcomponents:httpcore-nio:jar:${httpcomponents.httpcore.version}</include> <include>org.apache.httpcomponents:httpcore:jar:${httpcomponents.httpcore.version}</include> <include>org.apache.httpcomponents:httpmime:jar:${httpcomponents.httpmime.version}</include> - <include>com.google.protobuf:protobuf-java:jar:${protobuf-java.version}</include> - <include>org.noggit:noggit:jar:${noggit.version}</include> - <include>com.fasterxml.jackson.core:jackson-annotations:jar:${fasterxml.jackson.version}</include> - <include>com.fasterxml.jackson.core:jackson-core:jar:${fasterxml.jackson.version}</include> - <include>com.fasterxml.jackson.core:jackson-databind:jar:${fasterxml.jackson.version}</include> - <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar:${fasterxml.jackson.version}</include> - <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar:${fasterxml.jackson.version}</include> <include>org.apache.solr:solr-solrj:jar:${solr.version}</include> - <include>commons-codec:commons-codec</include> <include>org.codehaus.woodstox:stax2-api</include> - <include>com.fasterxml.woodstox:woodstox-core</include> - <include>org.elasticsearch:elasticsearch</include> - <include>org.elasticsearch:elasticsearch-core</include> - <include>org.elasticsearch:elasticsearch-x-content</include> - <include>org.elasticsearch.client:elasticsearch-rest-client</include> - <include>org.elasticsearch.client:elasticsearch-rest-high-level-client</include> - <include>org.elasticsearch.plugin:rank-eval-client</include> - <include>org.elasticsearch.plugin:lang-mustache-client</include> - <include>org.apache.commons:commons-lang3:jar:${commons.lang3.version}</include> - <include>org.apache.httpcomponents:httpcore-nio:jar:${httpcomponents.httpcore.version}</include> - <include>org.apache.httpcomponents:httpasyncclient:jar:${httpcomponents.httpasyncclient.version}</include> - <include>org.apache.lucene:lucene-core</include> - <include>joda-time:joda-time</include> - <include>com.carrotsearch:hppc</include> - <include>org.apache.hive:hive-storage-api:jar:${hive.storage-api.version}</include> - <include>org.apache.orc:orc-core:jar:${orc.version}</include> - <include>org.apache.orc:orc-shims:jar:${orc.version}</include> - <include>io.airlift:aircompressor:jar:${aircompressor.version}</include> - <include>org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version}</include> + <include>org.eclipse.jetty:jetty-client:jar:${jetty-client.version}</include> </includes> + <excludes> + <exclude>com.fasterxml.jackson.core:*</exclude> + <exclude>com.fasterxml.jackson.jaxrs:*</exclude> + <exclude>com.fasterxml.jackson.module:*</exclude> + <exclude>com.fasterxml.jackson.dataformat:*</exclude> + <exclude>org.apache.kafka:kafka-clients</exclude> + <exclude>org.glassfish.jersey.core:*</exclude> + <exclude>org.glassfish.jersey.ext:*</exclude> + <exclude>org.glassfish.jersey.inject:*</exclude> + <exclude>org.glassfish.jersey.media:*</exclude> + <exclude>org.glassfish.hk2:*</exclude> + <exclude>javax.ws.rs:javax.ws.rs-api</exclude> + <exclude>jakarta.ws.rs:jakarta.ws.rs-api</exclude> + <exclude>org.slf4j:*</exclude> + <exclude>org.apache.logging.log4j:*</exclude> + <exclude>ch.qos.reload4j:*</exclude> + </excludes> </binaries> </moduleSet> @@ -130,7 +129,6 @@ <include>org.apache.hadoop:hadoop-client-api:jar:${hadoop.version}</include> <include>org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version}</include> <include>org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version}</include> - <include>org.apache.ranger:ranger-plugins-cred</include> <include>org.codehaus.woodstox:stax2-api</include> <include>org.slf4j:slf4j-api:jar:${slf4j-api.version}</include> </includes> diff --git a/distro/src/main/assembly/plugin-kms.xml b/distro/src/main/assembly/plugin-kms.xml index ff0de24b6..d3bc2daf3 100755 --- a/distro/src/main/assembly/plugin-kms.xml +++ b/distro/src/main/assembly/plugin-kms.xml @@ -42,8 +42,7 @@ <useAllReactorProjects>true</useAllReactorProjects> <includes> <include>org.apache.ranger:ranger-audit-core</include> - <include>org.apache.ranger:ranger-audit-dest-hdfs</include> - <include>org.apache.ranger:ranger-audit-dest-solr</include> + <include>org.apache.ranger:ranger-audit-dest-auditserver</include> <include>org.apache.ranger:ranger-authz-api</include> <include>org.apache.ranger:ranger-common-utils</include> <include>org.apache.ranger:ranger-plugins-cred</include> @@ -61,6 +60,11 @@ <include>org.apache.commons:commons-configuration2:jar:${commons.configuration.version}</include> <include>org.apache.hadoop:hadoop-common:jar:${hadoop.version}</include> <include>com.google.code.gson:gson</include> + <include>com.sun.jersey:jersey-bundle:jar:${jersey-bundle.version}</include> + <include>javax.ws.rs:jsr311-api:jar:${jsr311-api.version}</include> + <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base</include> + <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider</include> + <include>com.fasterxml.jackson.module:jackson-module-jaxb-annotations</include> <include>org.eclipse.jetty:jetty-client:jar:${jetty-client.version}</include> <include>commons-collections:commons-collections</include> <include>commons-logging:commons-logging:jar:${commons.logging.version}</include> @@ -76,6 +80,16 @@ <include>io.airlift:aircompressor:jar:${aircompressor.version}</include> <include>org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version}</include> </includes> + <excludes> + <exclude>com.fasterxml.jackson.core:*</exclude> + <exclude>com.fasterxml.jackson.dataformat:*</exclude> + <exclude>org.glassfish.jersey.core:*</exclude> + <exclude>org.glassfish.jersey.ext:*</exclude> + <exclude>org.glassfish.jersey.inject:*</exclude> + <exclude>org.glassfish.jersey.media:*</exclude> + <exclude>org.glassfish.hk2:*</exclude> + <exclude>jakarta.ws.rs:jakarta.ws.rs-api</exclude> + </excludes> </binaries> </moduleSet> diff --git a/distro/src/main/assembly/plugin-kylin.xml b/distro/src/main/assembly/plugin-kylin.xml index fd60b98b6..c7562b6f4 100644 --- a/distro/src/main/assembly/plugin-kylin.xml +++ b/distro/src/main/assembly/plugin-kylin.xml @@ -42,8 +42,7 @@ <useAllReactorProjects>true</useAllReactorProjects> <includes> <include>org.apache.ranger:ranger-audit-core</include> - <include>org.apache.ranger:ranger-audit-dest-hdfs</include> - <include>org.apache.ranger:ranger-audit-dest-solr</include> + <include>org.apache.ranger:ranger-audit-dest-auditserver</include> <include>org.apache.ranger:ranger-authz-api</include> <include>org.apache.ranger:ranger-common-utils</include> <include>org.apache.ranger:ranger-plugins-cred</include> diff --git a/distro/src/main/assembly/plugin-ozone.xml b/distro/src/main/assembly/plugin-ozone.xml index a5bd0091d..aba01db6e 100644 --- a/distro/src/main/assembly/plugin-ozone.xml +++ b/distro/src/main/assembly/plugin-ozone.xml @@ -50,14 +50,9 @@ <unpack>false</unpack> <includes> <include>com.fasterxml.woodstox:woodstox-core:jar:${fasterxml.woodstox.version}</include> - <include>com.sun.jersey:jersey-client:jar:${jersey-bundle.version}</include> - <include>com.sun.jersey:jersey-core:jar:${jersey-bundle.version}</include> - <include>javax.ws.rs:jsr311-api:jar:${jsr311-api.version}</include> <include>commons-cli:commons-cli</include> <include>commons-collections:commons-collections</include> - <include>commons-configuration:commons-configuration:jar:${commons.configuration1.version}</include> <include>commons-io:commons-io:jar:${commons.io.version}</include> - <include>commons-logging:commons-logging:jar:${commons.logging.version}</include> <include>org.apache.commons:commons-collections4:jar:${commons.collections4.version}</include> <include>org.apache.commons:commons-compress:jar:${commons.compress.version}</include> <include>org.apache.commons:commons-configuration2:jar:${commons.configuration.version}</include> @@ -76,8 +71,7 @@ <useAllReactorProjects>true</useAllReactorProjects> <includes> <include>org.apache.ranger:ranger-audit-core</include> - <include>org.apache.ranger:ranger-audit-dest-hdfs</include> - <include>org.apache.ranger:ranger-audit-dest-solr</include> + <include>org.apache.ranger:ranger-audit-dest-auditserver</include> <include>org.apache.ranger:ranger-authz-api</include> <include>org.apache.ranger:ranger-common-utils</include> <include>org.apache.ranger:ranger-plugins-cred</include> @@ -94,61 +88,50 @@ <includes> <include>commons-collections:commons-collections</include> <include>org.apache.commons:commons-configuration2:jar:${commons.configuration.version}</include> - <include>commons-configuration:commons-configuration:jar:${commons.configuration1.version}</include> <include>org.eclipse.jetty:jetty-client:jar:${jetty-client.version}</include> <include>org.apache.httpcomponents:httpmime:jar:${httpcomponents.httpmime.version}</include> <include>org.apache.httpcomponents:httpclient:jar:${httpcomponents.httpclient.version}</include> <include>org.apache.httpcomponents:httpcore:jar:${httpcomponents.httpcore.version}</include> <include>org.apache.zookeeper:zookeeper:jar:${zookeeper.version}</include> <include>org.apache.zookeeper:zookeeper-jute:jar:${zookeeper.version}</include> - <include>org.noggit:noggit:jar:${noggit.version}</include> <include>org.apache.solr:solr-solrj:jar:${solr.version}</include> <include>com.fasterxml.woodstox:woodstox-core:jar:${fasterxml.woodstox.version}</include> <include>org.codehaus.woodstox:stax2-api:jar:${codehaus.woodstox.stax2api.version}</include> + <include>com.fasterxml.jackson.core:jackson-annotations:jar:${fasterxml.jackson.version}</include> + <include>com.fasterxml.jackson.core:jackson-core:jar:${fasterxml.jackson.version}</include> + <include>com.fasterxml.jackson.core:jackson-databind:jar:${fasterxml.jackson.version}</include> + <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar:${fasterxml.jackson.version}</include> + <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar:${fasterxml.jackson.version}</include> <include>com.sun.xml.bind:jaxb-impl</include> - <include>org.elasticsearch:elasticsearch</include> - <include>org.elasticsearch:elasticsearch-core</include> - <include>org.elasticsearch:elasticsearch-x-content</include> - <include>org.elasticsearch.client:elasticsearch-rest-client</include> - <include>org.elasticsearch.client:elasticsearch-rest-high-level-client</include> - <include>org.elasticsearch.plugin:rank-eval-client</include> - <include>org.elasticsearch.plugin:lang-mustache-client</include> - <include>org.apache.commons:commons-lang3:jar:${commons.lang3.version}</include> <include>org.apache.httpcomponents:httpcore-nio:jar:${httpcomponents.httpcore.version}</include> <include>org.apache.httpcomponents:httpasyncclient:jar:${httpcomponents.httpasyncclient.version}</include> - <include>org.apache.lucene:lucene-core</include> <include>joda-time:joda-time</include> <include>com.carrotsearch:hppc</include> - <include>org.apache.hive:hive-storage-api:jar:${hive.storage-api.version}</include> - <include>org.apache.orc:orc-core:jar:${orc.version}</include> - <include>org.apache.orc:orc-shims:jar:${orc.version}</include> <include>io.airlift:aircompressor:jar:${aircompressor.version}</include> - <include>javax.annotation:javax.annotation-api:jar:${javax.annotation-api}</include> + <include>javax.inject:javax.inject</include> + <include>javax.ws.rs:javax.ws.rs-api</include> <include>org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version}</include> + <include>org.glassfish.hk2:hk2-api</include> + <include>org.glassfish.hk2:hk2-locator</include> + <include>org.glassfish.hk2:hk2-utils</include> + <include>org.glassfish.hk2.external:aopalliance-repackaged</include> + <include>org.glassfish.hk2.external:jakarta.inject</include> + <include>org.glassfish.hk2.external:javax.inject</include> + <include>org.glassfish:osgi-resource-locator</include> + <include>org.glassfish.jersey.core:jersey-client</include> + <include>org.glassfish.jersey.core:jersey-common</include> + <include>org.glassfish.jersey.ext:jersey-entity-filtering:jar:${jersey-client.version}</include> + <include>org.glassfish.jersey.inject:jersey-hk2</include> + <include>org.glassfish.jersey.media:jersey-media-json-jackson:jar:${jersey-client.version}</include> + <include>com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar:${fasterxml.jackson.version}</include> </includes> - <excludes> - <exclude>com.fasterxml.jackson.core:*</exclude> - <exclude>com.fasterxml.jackson.jaxrs:*</exclude> - <exclude>com.fasterxml.jackson.module:*</exclude> - <exclude>com.fasterxml.jackson.dataformat:*</exclude> - <exclude>org.glassfish.jersey.core:*</exclude> - <exclude>org.glassfish.jersey.ext:*</exclude> - <exclude>org.glassfish.jersey.inject:*</exclude> - <exclude>org.glassfish.jersey.media:*</exclude> - <exclude>org.glassfish.hk2:*</exclude> - <exclude>javax.ws.rs:javax.ws.rs-api</exclude> - <exclude>jakarta.ws.rs:jakarta.ws.rs-api</exclude> - <exclude>org.slf4j:*</exclude> - <exclude>org.apache.logging.log4j:*</exclude> - <exclude>ch.qos.reload4j:*</exclude> - </excludes> </binaries> </moduleSet> </moduleSets> <fileSets> <fileSet> - <outputDirectory> install/conf.templates/enable</outputDirectory> + <outputDirectory>install/conf.templates/enable</outputDirectory> <directory>../plugin-ozone/conf</directory> <excludes> <exclude>*.sh</exclude> diff --git a/distro/src/main/assembly/plugin-presto.xml b/distro/src/main/assembly/plugin-presto.xml index de9bd7f8d..97cab759c 100644 --- a/distro/src/main/assembly/plugin-presto.xml +++ b/distro/src/main/assembly/plugin-presto.xml @@ -54,8 +54,7 @@ <useAllReactorProjects>true</useAllReactorProjects> <includes> <include>org.apache.ranger:ranger-audit-core</include> - <include>org.apache.ranger:ranger-audit-dest-hdfs</include> - <include>org.apache.ranger:ranger-audit-dest-solr</include> + <include>org.apache.ranger:ranger-audit-dest-auditserver</include> <include>org.apache.ranger:ranger-authz-api</include> <include>org.apache.ranger:ranger-common-utils</include> <include>org.apache.ranger:ranger-plugins-cred</include> diff --git a/distro/src/main/assembly/plugin-solr.xml b/distro/src/main/assembly/plugin-solr.xml index 221adba10..8018d3171 100644 --- a/distro/src/main/assembly/plugin-solr.xml +++ b/distro/src/main/assembly/plugin-solr.xml @@ -37,8 +37,7 @@ <useAllReactorProjects>true</useAllReactorProjects> <includes> <include>org.apache.ranger:ranger-audit-core</include> - <include>org.apache.ranger:ranger-audit-dest-hdfs</include> - <include>org.apache.ranger:ranger-audit-dest-solr</include> + <include>org.apache.ranger:ranger-audit-dest-auditserver</include> <include>org.apache.ranger:ranger-authz-api</include> <include>org.apache.ranger:ranger-common-utils</include> <include>org.apache.ranger:ranger-plugins-cred</include> @@ -53,10 +52,14 @@ <directoryMode>755</directoryMode> <fileMode>644</fileMode> <includes> + <include>org.apache.ranger:ranger-audit-core</include> + <include>org.apache.ranger:ranger-audit-dest-auditserver</include> <include>commons-collections:commons-collections</include> <include>com.google.code.gson:gson</include> - <include>com.sun.jersey:jersey-client:jar:${jersey-bundle.version}</include> - <include>com.sun.jersey:jersey-core:jar:${jersey-bundle.version}</include> + <include>com.sun.jersey:jersey-bundle:jar:${jersey-bundle.version}</include> + <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar:${fasterxml.jackson.version}</include> + <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar:${fasterxml.jackson.version}</include> + <include>com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar:${fasterxml.jackson.version}</include> <include>javax.ws.rs:jsr311-api:jar:${jsr311-api.version}</include> <include>org.apache.hadoop:hadoop-client-api:jar:${hadoop.version}</include> <include>org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version}</include> diff --git a/distro/src/main/assembly/plugin-sqoop.xml b/distro/src/main/assembly/plugin-sqoop.xml index d01add360..a9182fb3c 100644 --- a/distro/src/main/assembly/plugin-sqoop.xml +++ b/distro/src/main/assembly/plugin-sqoop.xml @@ -42,8 +42,7 @@ <useAllReactorProjects>true</useAllReactorProjects> <includes> <include>org.apache.ranger:ranger-audit-core</include> - <include>org.apache.ranger:ranger-audit-dest-hdfs</include> - <include>org.apache.ranger:ranger-audit-dest-solr</include> + <include>org.apache.ranger:ranger-audit-dest-auditserver</include> <include>org.apache.ranger:ranger-authz-api</include> <include>org.apache.ranger:ranger-common-utils</include> <include>org.apache.ranger:ranger-plugins-cred</include> diff --git a/distro/src/main/assembly/plugin-trino.xml b/distro/src/main/assembly/plugin-trino.xml index 213555631..cfaca5fc8 100644 --- a/distro/src/main/assembly/plugin-trino.xml +++ b/distro/src/main/assembly/plugin-trino.xml @@ -27,8 +27,7 @@ <useAllReactorProjects>true</useAllReactorProjects> <includes> <include>org.apache.ranger:ranger-audit-core</include> - <include>org.apache.ranger:ranger-audit-dest-hdfs</include> - <include>org.apache.ranger:ranger-audit-dest-solr</include> + <include>org.apache.ranger:ranger-audit-dest-auditserver</include> <include>org.apache.ranger:ranger-authz-api</include> <include>org.apache.ranger:ranger-common-utils</include> <include>org.apache.ranger:ranger-plugins-cred</include> diff --git a/distro/src/main/assembly/plugin-yarn.xml b/distro/src/main/assembly/plugin-yarn.xml index 8e43e1c60..e91295b92 100644 --- a/distro/src/main/assembly/plugin-yarn.xml +++ b/distro/src/main/assembly/plugin-yarn.xml @@ -42,8 +42,7 @@ <useAllReactorProjects>true</useAllReactorProjects> <includes> <include>org.apache.ranger:ranger-audit-core</include> - <include>org.apache.ranger:ranger-audit-dest-hdfs</include> - <include>org.apache.ranger:ranger-audit-dest-solr</include> + <include>org.apache.ranger:ranger-audit-dest-auditserver</include> <include>org.apache.ranger:ranger-authz-api</include> <include>org.apache.ranger:ranger-common-utils</include> <include>org.apache.ranger:ranger-plugins-cred</include> @@ -58,31 +57,37 @@ <directoryMode>755</directoryMode> <fileMode>644</fileMode> <includes> + <include>com.carrotsearch:hppc</include> + <include>com.fasterxml.jackson.core:jackson-annotations:jar:${fasterxml.jackson.version}</include> + <include>com.fasterxml.jackson.core:jackson-core:jar:${fasterxml.jackson.version}</include> + <include>com.fasterxml.jackson.core:jackson-databind:jar:${fasterxml.jackson.databind.version}</include> + <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar:${fasterxml.jackson.version}</include> + <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar:${fasterxml.jackson.version}</include> + <include>com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar:${fasterxml.jackson.version}</include> + <include>org.slf4j:slf4j-api:jar:${slf4j-api.version}</include> <include>commons-collections:commons-collections</include> - <include>org.eclipse.jetty:jetty-client:jar:${jetty-client.version}</include> - <include>org.apache.commons:commons-lang3:jar:${commons.lang3.version}</include> - <include>org.apache.httpcomponents:httpmime:jar:${httpcomponents.httpmime.version}</include> + <include>io.airlift:aircompressor:jar:${aircompressor.version}</include> + <include>jakarta.ws.rs:jakarta.ws.rs-api</include> + <include>javax.annotation:javax.annotation-api</include> + <include>joda-time:joda-time</include> + <include>org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version}</include> + <include>org.apache.httpcomponents:httpasyncclient:jar:${httpcomponents.httpasyncclient.version}</include> <include>org.apache.httpcomponents:httpclient:jar:${httpcomponents.httpclient.version}</include> + <include>org.apache.httpcomponents:httpcore-nio:jar:${httpcomponents.httpcore.version}</include> <include>org.apache.httpcomponents:httpcore:jar:${httpcomponents.httpcore.version}</include> - <include>org.noggit:noggit:jar:${noggit.version}</include> + <include>org.apache.httpcomponents:httpmime:jar:${httpcomponents.httpmime.version}</include> <include>org.apache.solr:solr-solrj:jar:${solr.version}</include> - <include>org.elasticsearch:elasticsearch</include> - <include>org.elasticsearch:elasticsearch-core</include> - <include>org.elasticsearch:elasticsearch-x-content</include> - <include>org.elasticsearch.client:elasticsearch-rest-client</include> - <include>org.elasticsearch.client:elasticsearch-rest-high-level-client</include> - <include>org.elasticsearch.plugin:rank-eval-client</include> - <include>org.elasticsearch.plugin:lang-mustache-client</include> - <include>org.apache.httpcomponents:httpcore-nio:jar:${httpcomponents.httpcore.version}</include> - <include>org.apache.httpcomponents:httpasyncclient:jar:${httpcomponents.httpasyncclient.version}</include> - <include>org.apache.lucene:lucene-core</include> - <include>joda-time:joda-time</include> - <include>com.carrotsearch:hppc</include> - <include>org.apache.hive:hive-storage-api:jar:${hive.storage-api.version}</include> - <include>org.apache.orc:orc-core:jar:${orc.version}</include> - <include>org.apache.orc:orc-shims:jar:${orc.version}</include> - <include>io.airlift:aircompressor:jar:${aircompressor.version}</include> - <include>org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version}</include> + <include>org.eclipse.jetty:jetty-client:jar:${jetty-client.version}</include> + <include>org.glassfish.jersey.core:jersey-client</include> + <include>org.glassfish.jersey.core:jersey-common</include> + <include>org.glassfish.jersey.media:jersey-media-json-jackson</include> + <include>org.glassfish.jersey.ext:jersey-entity-filtering</include> + <include>org.graalvm.js:js:jar:${graalvm.version}</include> + <include>org.graalvm.js:js-scriptengine:jar:${graalvm.version}</include> + <include>org.graalvm.regex:regex:jar:${graalvm.version}</include> + <include>org.graalvm.sdk:graal-sdk:jar:${graalvm.version}</include> + <include>org.graalvm.truffle:truffle-api:jar:${graalvm.version}</include> + <include>com.ibm.icu:icu4j</include> </includes> </binaries> </moduleSet> @@ -104,17 +109,14 @@ <include>commons-cli:commons-cli</include> <include>commons-collections:commons-collections</include> <include>commons-io:commons-io:jar:${commons.io.version}</include> - <include>commons-logging:commons-logging:jar:${commons.logging.version}</include> <include>org.apache.commons:commons-collections4:jar:${commons.collections4.version}</include> <include>org.apache.commons:commons-compress:jar:${commons.compress.version}</include> <include>org.apache.commons:commons-configuration2:jar:${commons.configuration.version}</include> <include>org.apache.commons:commons-lang3:jar:${commons.lang3.version}</include> + <include>org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version}</include> <include>org.apache.hadoop:hadoop-auth:jar:${hadoop.version}</include> <include>org.apache.hadoop:hadoop-client-api:jar:${hadoop.version}</include> <include>org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version}</include> - <include>org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version}</include> - <include>org.apache.ranger:credentialbuilder</include> - <include>org.apache.ranger:ranger-plugins-cred</include> <include>org.codehaus.woodstox:stax2-api</include> <include>org.slf4j:slf4j-api:jar:${slf4j-api.version}</include> </includes> diff --git a/distro/src/main/assembly/storm-agent.xml b/distro/src/main/assembly/storm-agent.xml index 0336340de..309074fa4 100644 --- a/distro/src/main/assembly/storm-agent.xml +++ b/distro/src/main/assembly/storm-agent.xml @@ -42,8 +42,7 @@ <useAllReactorProjects>true</useAllReactorProjects> <includes> <include>org.apache.ranger:ranger-audit-core</include> - <include>org.apache.ranger:ranger-audit-dest-hdfs</include> - <include>org.apache.ranger:ranger-audit-dest-solr</include> + <include>org.apache.ranger:ranger-audit-dest-auditserver</include> <include>org.apache.ranger:ranger-authz-api</include> <include>org.apache.ranger:ranger-common-utils</include> <include>org.apache.ranger:ranger-plugins-cred</include> diff --git a/hbase-agent/conf/ranger-hbase-audit-changes.cfg b/hbase-agent/conf/ranger-hbase-audit-changes.cfg index 15d19e580..6096f9397 100644 --- a/hbase-agent/conf/ranger-hbase-audit-changes.cfg +++ b/hbase-agent/conf/ranger-hbase-audit-changes.cfg @@ -91,3 +91,8 @@ xasecure.audit.log4j.async.max.queue.size %XAAUDIT.LOG4J.ASYNC.MA xasecure.audit.log4j.async.max.flush.interval.ms %XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS% mod create-if-not-exists xasecure.audit.destination.log4j %XAAUDIT.LOG4J.DESTINATION.LOG4J% mod create-if-not-exists xasecure.audit.destination.log4j.logger %XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER% mod create-if-not-exists + +#Audit Service Destination +xasecure.audit.destination.auditserver %XAAUDIT.AUDITSERVER.ENABLE% mod create-if-not-exists +xasecure.audit.destination.auditserver.url %XAAUDIT.AUDITSERVER.URL% mod create-if-not-exists +xasecure.audit.destination.auditserver.batch.filespool.dir %XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/hbase-agent/pom.xml b/hbase-agent/pom.xml index 06f35cbf1..66957e4bc 100644 --- a/hbase-agent/pom.xml +++ b/hbase-agent/pom.xml @@ -54,21 +54,8 @@ </dependency> <dependency> <groupId>com.sun.jersey</groupId> - <artifactId>jersey-client</artifactId> + <artifactId>jersey-bundle</artifactId> <version>${jersey-bundle.version}</version> - <type>jar</type> - </dependency> - <dependency> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-core</artifactId> - <version>${jersey-bundle.version}</version> - <type>jar</type> - <exclusions> - <exclusion> - <groupId>javax.ws.rs</groupId> - <artifactId>jsr311-api</artifactId> - </exclusion> - </exclusions> </dependency> <dependency> <groupId>io.opentelemetry</groupId> @@ -175,18 +162,7 @@ </dependency> <dependency> <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-hdfs</artifactId> - <version>${project.version}</version> - <exclusions> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>*</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-solr</artifactId> + <artifactId>ranger-audit-dest-auditserver</artifactId> <version>${project.version}</version> </dependency> <dependency> diff --git a/hbase-agent/scripts/install.properties b/hbase-agent/scripts/install.properties index 4ad1269b0..555145313 100644 --- a/hbase-agent/scripts/install.properties +++ b/hbase-agent/scripts/install.properties @@ -122,6 +122,10 @@ XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE +#Audit Server Provider +XAAUDIT.AUDITSERVER.ENABLE=false +XAAUDIT.AUDITSERVER.URL=http://ranger-audit:7081 +XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR=/var/log/hive/audit/http/spool # End of V3 properties # diff --git a/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg b/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg index 49244c66b..3be456cd4 100644 --- a/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg +++ b/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg @@ -88,3 +88,8 @@ xasecure.audit.log4j.async.max.queue.size %XAAUDIT.LOG4J.ASYNC.MA xasecure.audit.log4j.async.max.flush.interval.ms %XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS% mod create-if-not-exists xasecure.audit.destination.log4j %XAAUDIT.LOG4J.DESTINATION.LOG4J% mod create-if-not-exists xasecure.audit.destination.log4j.logger %XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER% mod create-if-not-exists + +#Audit Service Destination +xasecure.audit.destination.auditserver %XAAUDIT.AUDITSERVER.ENABLE% mod create-if-not-exists +xasecure.audit.destination.auditserver.url %XAAUDIT.AUDITSERVER.URL% mod create-if-not-exists +xasecure.audit.destination.auditserver.batch.filespool.dir %XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/hdfs-agent/conf/ranger-hdfs-audit.xml b/hdfs-agent/conf/ranger-hdfs-audit.xml index e16ea590d..2b8d8556c 100644 --- a/hdfs-agent/conf/ranger-hdfs-audit.xml +++ b/hdfs-agent/conf/ranger-hdfs-audit.xml @@ -211,6 +211,21 @@ <property> <name>xasecure.audit.solr.solr_url</name> <value>http://localhost:6083/solr/ranger_audits</value> - </property> - + </property> + + <!-- Audit Server destination --> + <property> + <name>xasecure.audit.destination.auditserver</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.destination.auditserver.url</name> + <value>http://localhost:7081</value> + </property> + + <property> + <name>xasecure.audit.destination.auditserver.batch.filespool.dir</name> + <value>/var/log/hive/audit/http/spool</value> + </property> </configuration> diff --git a/hdfs-agent/pom.xml b/hdfs-agent/pom.xml index 0dc71890b..ff1b40f49 100644 --- a/hdfs-agent/pom.xml +++ b/hdfs-agent/pom.xml @@ -36,6 +36,11 @@ <artifactId>netty-all</artifactId> <version>${netty-all.version}</version> </dependency> + <dependency> + <groupId>javax.annotation</groupId> + <artifactId>javax.annotation-api</artifactId> + <version>${javax.annotation-api.version}</version> + </dependency> <dependency> <groupId>javax.servlet</groupId> <artifactId>javax.servlet-api</artifactId> @@ -116,12 +121,7 @@ </dependency> <dependency> <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-hdfs</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-solr</artifactId> + <artifactId>ranger-audit-dest-auditserver</artifactId> <version>${project.version}</version> </dependency> <dependency> diff --git a/hdfs-agent/scripts/install.properties b/hdfs-agent/scripts/install.properties index fedecc1d2..20deb540c 100644 --- a/hdfs-agent/scripts/install.properties +++ b/hdfs-agent/scripts/install.properties @@ -116,6 +116,10 @@ XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE +#Audit Server Provider +XAAUDIT.AUDITSERVER.ENABLE=false +XAAUDIT.AUDITSERVER.URL=http://ranger-audit:7081 +XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR=/var/log/hive/audit/http/spool # End of V3 properties # diff --git a/hive-agent/conf/ranger-hive-audit-changes.cfg b/hive-agent/conf/ranger-hive-audit-changes.cfg index 651372d3e..c81a52746 100644 --- a/hive-agent/conf/ranger-hive-audit-changes.cfg +++ b/hive-agent/conf/ranger-hive-audit-changes.cfg @@ -89,3 +89,8 @@ xasecure.audit.log4j.async.max.queue.size %XAAUDIT.LOG4J.ASYNC.MA xasecure.audit.log4j.async.max.flush.interval.ms %XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS% mod create-if-not-exists xasecure.audit.destination.log4j %XAAUDIT.LOG4J.DESTINATION.LOG4J% mod create-if-not-exists xasecure.audit.destination.log4j.logger %XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER% mod create-if-not-exists + +#Audit Service Destination +xasecure.audit.destination.auditserver %XAAUDIT.AUDITSERVER.ENABLE% mod create-if-not-exists +xasecure.audit.destination.auditserver.url %XAAUDIT.AUDITSERVER.URL% mod create-if-not-exists +xasecure.audit.destination.auditserver.batch.filespool.dir %XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/hive-agent/conf/ranger-hive-audit.xml b/hive-agent/conf/ranger-hive-audit.xml index d42e58935..5693a1be1 100644 --- a/hive-agent/conf/ranger-hive-audit.xml +++ b/hive-agent/conf/ranger-hive-audit.xml @@ -211,6 +211,21 @@ <property> <name>xasecure.audit.solr.solr_url</name> <value>http://localhost:6083/solr/ranger_audits</value> - </property> - + </property> + + <!-- Audit Server HTTP destination --> + <property> + <name>xasecure.audit.destination.auditserver</name> + <value>true</value> + </property> + + <property> + <name>xasecure.audit.destination.auditserver.url</name> + <value>http://localhost:7081</value> + </property> + + <property> + <name>xasecure.audit.destination.auditserver.batch.filespool.dir</name> + <value>/var/log/hive/audit/http/spool</value> + </property> </configuration> diff --git a/hive-agent/pom.xml b/hive-agent/pom.xml index 8318ef352..1674c553a 100644 --- a/hive-agent/pom.xml +++ b/hive-agent/pom.xml @@ -184,12 +184,7 @@ </dependency> <dependency> <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-hdfs</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-solr</artifactId> + <artifactId>ranger-audit-dest-auditserver</artifactId> <version>${project.version}</version> </dependency> <dependency> diff --git a/hive-agent/scripts/install.properties b/hive-agent/scripts/install.properties index 4d8976b6d..8ac29281d 100644 --- a/hive-agent/scripts/install.properties +++ b/hive-agent/scripts/install.properties @@ -119,6 +119,10 @@ XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE +#Audit Server Provider +XAAUDIT.AUDITSERVER.ENABLE=false +XAAUDIT.AUDITSERVER.URL=http://ranger-audit:7081 +XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR=/var/log/hive/audit/http/spool # End of V3 properties diff --git a/kms/pom.xml b/kms/pom.xml index 424f4df22..7b3a9ad43 100644 --- a/kms/pom.xml +++ b/kms/pom.xml @@ -93,21 +93,6 @@ </exclusion> </exclusions> </dependency> - <dependency> - <groupId>com.google.api</groupId> - <artifactId>gax</artifactId> - <version>${google.gax.version}</version> - </dependency> - <dependency> - <groupId>com.google.api</groupId> - <artifactId>gax-grpc</artifactId> - <version>${google.gax.version}</version> - </dependency> - <dependency> - <groupId>com.google.api</groupId> - <artifactId>gax-httpjson</artifactId> - <version>${google.gax.version}</version> - </dependency> <dependency> <groupId>com.google.cloud</groupId> <artifactId>google-cloud-kms</artifactId> diff --git a/kms/scripts/install.properties b/kms/scripts/install.properties index 499ebbaec..c8b8d5f0e 100755 --- a/kms/scripts/install.properties +++ b/kms/scripts/install.properties @@ -264,6 +264,10 @@ XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE +#Audit Server Provider +XAAUDIT.AUDITSERVER.ENABLE=false +XAAUDIT.AUDITSERVER.URL=http://ranger-audit:7081 +XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR=/var/log/hive/audit/http/spool # End of V3 properties diff --git a/knox-agent/conf/ranger-knox-audit-changes.cfg b/knox-agent/conf/ranger-knox-audit-changes.cfg index 651372d3e..c81a52746 100644 --- a/knox-agent/conf/ranger-knox-audit-changes.cfg +++ b/knox-agent/conf/ranger-knox-audit-changes.cfg @@ -89,3 +89,8 @@ xasecure.audit.log4j.async.max.queue.size %XAAUDIT.LOG4J.ASYNC.MA xasecure.audit.log4j.async.max.flush.interval.ms %XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS% mod create-if-not-exists xasecure.audit.destination.log4j %XAAUDIT.LOG4J.DESTINATION.LOG4J% mod create-if-not-exists xasecure.audit.destination.log4j.logger %XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER% mod create-if-not-exists + +#Audit Service Destination +xasecure.audit.destination.auditserver %XAAUDIT.AUDITSERVER.ENABLE% mod create-if-not-exists +xasecure.audit.destination.auditserver.url %XAAUDIT.AUDITSERVER.URL% mod create-if-not-exists +xasecure.audit.destination.auditserver.batch.filespool.dir %XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/knox-agent/pom.xml b/knox-agent/pom.xml index 6394c191a..607f1127a 100644 --- a/knox-agent/pom.xml +++ b/knox-agent/pom.xml @@ -50,6 +50,17 @@ <artifactId>protobuf-java</artifactId> <version>${protobuf-java.version}</version> </dependency> + <dependency> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-client</artifactId> + <version>${jersey-bundle.version}</version> + <exclusions> + <exclusion> + <groupId>javax.ws.rs</groupId> + <artifactId>jsr311-api</artifactId> + </exclusion> + </exclusions> + </dependency> <dependency> <groupId>commons-collections</groupId> <artifactId>commons-collections</artifactId> @@ -65,6 +76,11 @@ <artifactId>javax.servlet-api</artifactId> <version>${javax.servlet.version}</version> </dependency> + <dependency> + <groupId>javax.ws.rs</groupId> + <artifactId>javax.ws.rs-api</artifactId> + <version>2.1.1</version> + </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client-api</artifactId> @@ -109,33 +125,23 @@ </dependency> <dependency> <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-hdfs</artifactId> - <version>${project.version}</version> - <!-- TO DEBUG JERSEY2 code: uncomment this exclusion - <exclusions><exclusion><artifactId>*</artifactId><groupId>*</groupId></exclusion></exclusions> - --> - </dependency> - <dependency> - <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-solr</artifactId> + <artifactId>ranger-audit-dest-auditserver</artifactId> <version>${project.version}</version> - <!-- TO DEBUG JERSEY2 code: uncomment this exclusion - <exclusions><exclusion><artifactId>*</artifactId><groupId>*</groupId></exclusion></exclusions> - --> </dependency> <dependency> <groupId>org.apache.ranger</groupId> <artifactId>ranger-plugins-common</artifactId> <version>${project.version}</version> <exclusions> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-bundle</artifactId> + </exclusion> <exclusion> <groupId>javax.ws.rs</groupId> <artifactId>jsr311-api</artifactId> </exclusion> </exclusions> - <!-- TO DEBUG JERSEY2 code: uncomment this exclusion - <exclusions><exclusion><artifactId>*</artifactId><groupId>*</groupId></exclusion></exclusions> - --> </dependency> <dependency> <groupId>org.glassfish.jersey.core</groupId> diff --git a/knox-agent/scripts/install.properties b/knox-agent/scripts/install.properties index d4cfa5202..85ef353e3 100644 --- a/knox-agent/scripts/install.properties +++ b/knox-agent/scripts/install.properties @@ -114,6 +114,10 @@ XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE +#Audit Server Provider +XAAUDIT.AUDITSERVER.ENABLE=false +XAAUDIT.AUDITSERVER.URL=http://ranger-audit:7081 +XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR=/var/log/hive/audit/http/spool # End of V3 properties diff --git a/plugin-atlas/conf/ranger-atlas-audit-changes.cfg b/plugin-atlas/conf/ranger-atlas-audit-changes.cfg index 59c3927e9..86e14684d 100644 --- a/plugin-atlas/conf/ranger-atlas-audit-changes.cfg +++ b/plugin-atlas/conf/ranger-atlas-audit-changes.cfg @@ -48,4 +48,9 @@ xasecure.audit.log4j.async.max.flush.interval.ms %XAAUDIT.LOG4J.A xasecure.audit.destination.log4j %XAAUDIT.LOG4J.DESTINATION.LOG4J% mod create-if-not-exists xasecure.audit.destination.log4j.logger %XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER% mod create-if-not-exists +#Audit Service Destination +xasecure.audit.destination.auditserver %XAAUDIT.AUDITSERVER.ENABLE% mod create-if-not-exists +xasecure.audit.destination.auditserver.url %XAAUDIT.AUDITSERVER.URL% mod create-if-not-exists +xasecure.audit.destination.auditserver.batch.filespool.dir %XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR% mod create-if-not-exists + diff --git a/plugin-atlas/pom.xml b/plugin-atlas/pom.xml index f30c930d9..c90bd41f4 100644 --- a/plugin-atlas/pom.xml +++ b/plugin-atlas/pom.xml @@ -81,12 +81,7 @@ </dependency> <dependency> <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-hdfs</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-solr</artifactId> + <artifactId>ranger-audit-dest-auditserver</artifactId> <version>${project.version}</version> </dependency> <dependency> diff --git a/plugin-atlas/scripts/install.properties b/plugin-atlas/scripts/install.properties index fdcdbd3ec..404e209b3 100644 --- a/plugin-atlas/scripts/install.properties +++ b/plugin-atlas/scripts/install.properties @@ -117,6 +117,10 @@ XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE +#Audit Server Provider +XAAUDIT.AUDITSERVER.ENABLE=false +XAAUDIT.AUDITSERVER.URL=http://ranger-audit:7081 +XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR=/var/log/hive/audit/http/spool # End of V3 properties # diff --git a/plugin-elasticsearch/conf/ranger-elasticsearch-audit-changes.cfg b/plugin-elasticsearch/conf/ranger-elasticsearch-audit-changes.cfg index 651372d3e..c81a52746 100644 --- a/plugin-elasticsearch/conf/ranger-elasticsearch-audit-changes.cfg +++ b/plugin-elasticsearch/conf/ranger-elasticsearch-audit-changes.cfg @@ -89,3 +89,8 @@ xasecure.audit.log4j.async.max.queue.size %XAAUDIT.LOG4J.ASYNC.MA xasecure.audit.log4j.async.max.flush.interval.ms %XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS% mod create-if-not-exists xasecure.audit.destination.log4j %XAAUDIT.LOG4J.DESTINATION.LOG4J% mod create-if-not-exists xasecure.audit.destination.log4j.logger %XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER% mod create-if-not-exists + +#Audit Service Destination +xasecure.audit.destination.auditserver %XAAUDIT.AUDITSERVER.ENABLE% mod create-if-not-exists +xasecure.audit.destination.auditserver.url %XAAUDIT.AUDITSERVER.URL% mod create-if-not-exists +xasecure.audit.destination.auditserver.batch.filespool.dir %XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/plugin-elasticsearch/pom.xml b/plugin-elasticsearch/pom.xml index 152fa855d..42e493a6f 100644 --- a/plugin-elasticsearch/pom.xml +++ b/plugin-elasticsearch/pom.xml @@ -48,12 +48,7 @@ </dependency> <dependency> <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-hdfs</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-solr</artifactId> + <artifactId>ranger-audit-dest-auditserver</artifactId> <version>${project.version}</version> </dependency> <dependency> diff --git a/plugin-elasticsearch/scripts/install.properties b/plugin-elasticsearch/scripts/install.properties index 971b7c797..35ee159ae 100644 --- a/plugin-elasticsearch/scripts/install.properties +++ b/plugin-elasticsearch/scripts/install.properties @@ -116,6 +116,10 @@ XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE +#Audit Server Provider +XAAUDIT.AUDITSERVER.ENABLE=false +XAAUDIT.AUDITSERVER.URL=http://ranger-audit:7081 +XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR=/var/log/hive/audit/http/spool # End of V3 properties # diff --git a/plugin-kafka/conf/ranger-kafka-audit-changes.cfg b/plugin-kafka/conf/ranger-kafka-audit-changes.cfg index 1903d98ca..ba06eddc4 100644 --- a/plugin-kafka/conf/ranger-kafka-audit-changes.cfg +++ b/plugin-kafka/conf/ranger-kafka-audit-changes.cfg @@ -82,3 +82,8 @@ xasecure.audit.log4j.async.max.queue.size %XAAUDIT.LOG4J.ASYNC.MA xasecure.audit.log4j.async.max.flush.interval.ms %XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS% mod create-if-not-exists xasecure.audit.destination.log4j %XAAUDIT.LOG4J.DESTINATION.LOG4J% mod create-if-not-exists xasecure.audit.destination.log4j.logger %XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER% mod create-if-not-exists + +#Audit Service Destination +xasecure.audit.destination.auditserver %XAAUDIT.AUDITSERVER.ENABLE% mod create-if-not-exists +xasecure.audit.destination.auditserver.url %XAAUDIT.AUDITSERVER.URL% mod create-if-not-exists +xasecure.audit.destination.auditserver.batch.filespool.dir %XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/plugin-kafka/pom.xml b/plugin-kafka/pom.xml index 5fcbeaba3..44effb25b 100644 --- a/plugin-kafka/pom.xml +++ b/plugin-kafka/pom.xml @@ -78,12 +78,7 @@ </dependency> <dependency> <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-hdfs</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-solr</artifactId> + <artifactId>ranger-audit-dest-auditserver</artifactId> <version>${project.version}</version> </dependency> <dependency> diff --git a/plugin-kafka/scripts/install.properties b/plugin-kafka/scripts/install.properties index 12736c5b7..a95023004 100644 --- a/plugin-kafka/scripts/install.properties +++ b/plugin-kafka/scripts/install.properties @@ -117,6 +117,10 @@ XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE +#Audit Server Provider +XAAUDIT.AUDITSERVER.ENABLE=false +XAAUDIT.AUDITSERVER.URL=http://ranger-audit:7081 +XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR=/var/log/hive/audit/http/spool # End of V3 properties # diff --git a/plugin-kafka/template/configuration.xml b/plugin-kafka/template/configuration.xml new file mode 100644 index 000000000..bdb012589 --- /dev/null +++ b/plugin-kafka/template/configuration.xml @@ -0,0 +1,20 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +<configuration> +</configuration> diff --git a/plugin-kms/conf/ranger-kms-audit-changes.cfg b/plugin-kms/conf/ranger-kms-audit-changes.cfg index d13d20288..c208975d3 100644 --- a/plugin-kms/conf/ranger-kms-audit-changes.cfg +++ b/plugin-kms/conf/ranger-kms-audit-changes.cfg @@ -83,3 +83,8 @@ xasecure.audit.destination.hdfs.config.fs.azure.account.keyprovider.%AZURE.ACCOU #xasecure.audit.destination.file %XAAUDIT.FILE.ENABLE% mod create-if-not-exists #xasecure.audit.destination.file.dir %XAAUDIT.FILE.DIR% mod create-if-not-exists + +#Audit Service Destination +xasecure.audit.destination.auditserver %XAAUDIT.AUDITSERVER.ENABLE% mod create-if-not-exists +xasecure.audit.destination.auditserver.url %XAAUDIT.AUDITSERVER.URL% mod create-if-not-exists +xasecure.audit.destination.auditserver.batch.filespool.dir %XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/plugin-kms/pom.xml b/plugin-kms/pom.xml index fc617e20a..88a7802c5 100644 --- a/plugin-kms/pom.xml +++ b/plugin-kms/pom.xml @@ -31,6 +31,16 @@ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> </properties> <dependencies> + <dependency> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-bundle</artifactId> + <version>${jersey-bundle.version}</version> + </dependency> + <dependency> + <groupId>javax.ws.rs</groupId> + <artifactId>jsr311-api</artifactId> + <version>${jsr311-api.version}</version> + </dependency> <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpcore</artifactId> @@ -38,12 +48,7 @@ </dependency> <dependency> <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-hdfs</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-solr</artifactId> + <artifactId>ranger-audit-dest-auditserver</artifactId> <version>${project.version}</version> </dependency> <dependency> diff --git a/plugin-kudu/pom.xml b/plugin-kudu/pom.xml index 555a94a4d..f6fadf640 100644 --- a/plugin-kudu/pom.xml +++ b/plugin-kudu/pom.xml @@ -38,12 +38,7 @@ </dependency> <dependency> <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-hdfs</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-solr</artifactId> + <artifactId>ranger-audit-dest-auditserver</artifactId> <version>${project.version}</version> </dependency> <dependency> diff --git a/plugin-kylin/conf/ranger-kylin-audit-changes.cfg b/plugin-kylin/conf/ranger-kylin-audit-changes.cfg index 651372d3e..c81a52746 100644 --- a/plugin-kylin/conf/ranger-kylin-audit-changes.cfg +++ b/plugin-kylin/conf/ranger-kylin-audit-changes.cfg @@ -89,3 +89,8 @@ xasecure.audit.log4j.async.max.queue.size %XAAUDIT.LOG4J.ASYNC.MA xasecure.audit.log4j.async.max.flush.interval.ms %XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS% mod create-if-not-exists xasecure.audit.destination.log4j %XAAUDIT.LOG4J.DESTINATION.LOG4J% mod create-if-not-exists xasecure.audit.destination.log4j.logger %XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER% mod create-if-not-exists + +#Audit Service Destination +xasecure.audit.destination.auditserver %XAAUDIT.AUDITSERVER.ENABLE% mod create-if-not-exists +xasecure.audit.destination.auditserver.url %XAAUDIT.AUDITSERVER.URL% mod create-if-not-exists +xasecure.audit.destination.auditserver.batch.filespool.dir %XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/plugin-kylin/pom.xml b/plugin-kylin/pom.xml index 2c742571a..5f8518e6d 100644 --- a/plugin-kylin/pom.xml +++ b/plugin-kylin/pom.xml @@ -48,12 +48,7 @@ </dependency> <dependency> <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-hdfs</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-solr</artifactId> + <artifactId>ranger-audit-dest-auditserver</artifactId> <version>${project.version}</version> </dependency> <dependency> @@ -130,12 +125,22 @@ <groupId>org.apache.kylin</groupId> <artifactId>kylin-shaded-guava</artifactId> </exclusion> + <exclusion> + <groupId>org.roaringbitmap</groupId> + <artifactId>RoaringBitmap</artifactId> + </exclusion> <exclusion> <groupId>org.slf4j</groupId> <artifactId>*</artifactId> </exclusion> </exclusions> </dependency> + <dependency> + <groupId>org.roaringbitmap</groupId> + <artifactId>RoaringBitmap</artifactId> + <version>0.7.36</version> + <scope>provided</scope> + </dependency> <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> diff --git a/plugin-kylin/scripts/install.properties b/plugin-kylin/scripts/install.properties index e807b483e..feda34784 100644 --- a/plugin-kylin/scripts/install.properties +++ b/plugin-kylin/scripts/install.properties @@ -116,6 +116,10 @@ XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE +#Audit Server Provider +XAAUDIT.AUDITSERVER.ENABLE=false +XAAUDIT.AUDITSERVER.URL=http://ranger-audit:7081 +XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR=/var/log/hive/audit/http/spool # End of V3 properties diff --git a/plugin-nifi-registry/pom.xml b/plugin-nifi-registry/pom.xml index b07ec4838..9d1b906c1 100644 --- a/plugin-nifi-registry/pom.xml +++ b/plugin-nifi-registry/pom.xml @@ -53,12 +53,7 @@ </dependency> <dependency> <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-hdfs</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-solr</artifactId> + <artifactId>ranger-audit-dest-auditserver</artifactId> <version>${project.version}</version> </dependency> <dependency> diff --git a/plugin-nifi/pom.xml b/plugin-nifi/pom.xml index 636708ead..52a10ce00 100644 --- a/plugin-nifi/pom.xml +++ b/plugin-nifi/pom.xml @@ -43,12 +43,7 @@ </dependency> <dependency> <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-hdfs</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-solr</artifactId> + <artifactId>ranger-audit-dest-auditserver</artifactId> <version>${project.version}</version> </dependency> <dependency> diff --git a/plugin-ozone/conf/ranger-ozone-audit-changes.cfg b/plugin-ozone/conf/ranger-ozone-audit-changes.cfg index cc15921cc..619f48d77 100644 --- a/plugin-ozone/conf/ranger-ozone-audit-changes.cfg +++ b/plugin-ozone/conf/ranger-ozone-audit-changes.cfg @@ -82,3 +82,8 @@ xasecure.audit.log4j.async.max.queue.size %XAAUDIT.LOG4J.ASYNC.MA xasecure.audit.log4j.async.max.flush.interval.ms %XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS% mod create-if-not-exists xasecure.audit.destination.log4j %XAAUDIT.LOG4J.DESTINATION.LOG4J% mod create-if-not-exists xasecure.audit.destination.log4j.logger %XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER% mod create-if-not-exists + +#Audit Service Destination +xasecure.audit.destination.auditserver %XAAUDIT.AUDITSERVER.ENABLE% mod create-if-not-exists +xasecure.audit.destination.auditserver.url %XAAUDIT.AUDITSERVER.URL% mod create-if-not-exists +xasecure.audit.destination.auditserver.batch.filespool.dir %XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/plugin-ozone/pom.xml b/plugin-ozone/pom.xml index b90abc573..02488cf8a 100644 --- a/plugin-ozone/pom.xml +++ b/plugin-ozone/pom.xml @@ -36,6 +36,16 @@ limitations under the License. <artifactId>jackson-databind</artifactId> <version>${fasterxml.jackson.version}</version> </dependency> + <dependency> + <groupId>com.fasterxml.jackson.jaxrs</groupId> + <artifactId>jackson-jaxrs-base</artifactId> + <version>${fasterxml.jackson.version}</version> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.module</groupId> + <artifactId>jackson-module-jaxb-annotations</artifactId> + <version>${fasterxml.jackson.version}</version> + </dependency> <dependency> <groupId>com.google.protobuf</groupId> <artifactId>protobuf-java</artifactId> @@ -102,12 +112,7 @@ limitations under the License. </dependency> <dependency> <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-hdfs</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-solr</artifactId> + <artifactId>ranger-audit-dest-auditserver</artifactId> <version>${project.version}</version> </dependency> <dependency> @@ -131,10 +136,22 @@ limitations under the License. <artifactId>bcpkix-jdk18on</artifactId> <version>${org.bouncycastle.bcpkix-jdk18on}</version> </dependency> + <dependency> + <groupId>com.fasterxml.jackson.jaxrs</groupId> + <artifactId>jackson-jaxrs-json-provider</artifactId> + <version>${fasterxml.jackson.version}</version> + <scope>runtime</scope> + <exclusions> + <exclusion> + <groupId>jakarta.activation</groupId> + <artifactId>jakarta.activation-api</artifactId> + </exclusion> + </exclusions> + </dependency> <dependency> <groupId>javax.inject</groupId> <artifactId>javax.inject</artifactId> - <version>${javax-inject.version}</version> + <version>${javax.inject.version}</version> <scope>runtime</scope> </dependency> <dependency> @@ -143,12 +160,6 @@ limitations under the License. <version>${junit.jupiter.version}</version> <scope>test</scope> </dependency> - <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-inline</artifactId> - <version>${mockito.version}</version> - <scope>test</scope> - </dependency> <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-junit-jupiter</artifactId> diff --git a/plugin-ozone/scripts/install.properties b/plugin-ozone/scripts/install.properties index 9285505a0..02c1f141f 100644 --- a/plugin-ozone/scripts/install.properties +++ b/plugin-ozone/scripts/install.properties @@ -117,6 +117,10 @@ XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE +#Audit Server Provider +XAAUDIT.AUDITSERVER.ENABLE=false +XAAUDIT.AUDITSERVER.URL=http://ranger-audit:7081 +XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR=/var/log/hive/audit/http/spool # End of V3 properties # diff --git a/plugin-presto/conf/ranger-presto-audit-changes.cfg b/plugin-presto/conf/ranger-presto-audit-changes.cfg index 1903d98ca..ba06eddc4 100644 --- a/plugin-presto/conf/ranger-presto-audit-changes.cfg +++ b/plugin-presto/conf/ranger-presto-audit-changes.cfg @@ -82,3 +82,8 @@ xasecure.audit.log4j.async.max.queue.size %XAAUDIT.LOG4J.ASYNC.MA xasecure.audit.log4j.async.max.flush.interval.ms %XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS% mod create-if-not-exists xasecure.audit.destination.log4j %XAAUDIT.LOG4J.DESTINATION.LOG4J% mod create-if-not-exists xasecure.audit.destination.log4j.logger %XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER% mod create-if-not-exists + +#Audit Service Destination +xasecure.audit.destination.auditserver %XAAUDIT.AUDITSERVER.ENABLE% mod create-if-not-exists +xasecure.audit.destination.auditserver.url %XAAUDIT.AUDITSERVER.URL% mod create-if-not-exists +xasecure.audit.destination.auditserver.batch.filespool.dir %XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/plugin-presto/pom.xml b/plugin-presto/pom.xml index ecd78530f..d534f44e4 100644 --- a/plugin-presto/pom.xml +++ b/plugin-presto/pom.xml @@ -63,12 +63,7 @@ </dependency> <dependency> <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-hdfs</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-solr</artifactId> + <artifactId>ranger-audit-dest-auditserver</artifactId> <version>${project.version}</version> </dependency> <dependency> diff --git a/plugin-presto/scripts/install.properties b/plugin-presto/scripts/install.properties index fa3984a1e..c5e091d2d 100644 --- a/plugin-presto/scripts/install.properties +++ b/plugin-presto/scripts/install.properties @@ -116,6 +116,10 @@ XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE +#Audit Server Provider +XAAUDIT.AUDITSERVER.ENABLE=false +XAAUDIT.AUDITSERVER.URL=http://ranger-audit:7081 +XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR=/var/log/hive/audit/http/spool # End of V3 properties diff --git a/plugin-solr/conf/ranger-solr-audit-changes.cfg b/plugin-solr/conf/ranger-solr-audit-changes.cfg index 651372d3e..c81a52746 100644 --- a/plugin-solr/conf/ranger-solr-audit-changes.cfg +++ b/plugin-solr/conf/ranger-solr-audit-changes.cfg @@ -89,3 +89,8 @@ xasecure.audit.log4j.async.max.queue.size %XAAUDIT.LOG4J.ASYNC.MA xasecure.audit.log4j.async.max.flush.interval.ms %XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS% mod create-if-not-exists xasecure.audit.destination.log4j %XAAUDIT.LOG4J.DESTINATION.LOG4J% mod create-if-not-exists xasecure.audit.destination.log4j.logger %XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER% mod create-if-not-exists + +#Audit Service Destination +xasecure.audit.destination.auditserver %XAAUDIT.AUDITSERVER.ENABLE% mod create-if-not-exists +xasecure.audit.destination.auditserver.url %XAAUDIT.AUDITSERVER.URL% mod create-if-not-exists +xasecure.audit.destination.auditserver.batch.filespool.dir %XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/plugin-solr/pom.xml b/plugin-solr/pom.xml index 99c31e2d0..9d7cbd3a5 100644 --- a/plugin-solr/pom.xml +++ b/plugin-solr/pom.xml @@ -43,12 +43,7 @@ </dependency> <dependency> <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-hdfs</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-solr</artifactId> + <artifactId>ranger-audit-dest-auditserver</artifactId> <version>${project.version}</version> </dependency> <dependency> diff --git a/plugin-solr/scripts/install.properties b/plugin-solr/scripts/install.properties index fb85fb2cb..3fdda8054 100644 --- a/plugin-solr/scripts/install.properties +++ b/plugin-solr/scripts/install.properties @@ -117,6 +117,10 @@ XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE +#Audit Server Provider +XAAUDIT.AUDITSERVER.ENABLE=false +XAAUDIT.AUDITSERVER.URL=http://ranger-audit:7081 +XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR=/var/log/hive/audit/http/spool # End of V3 properties # diff --git a/plugin-sqoop/conf/ranger-sqoop-audit-changes.cfg b/plugin-sqoop/conf/ranger-sqoop-audit-changes.cfg index 651372d3e..c81a52746 100644 --- a/plugin-sqoop/conf/ranger-sqoop-audit-changes.cfg +++ b/plugin-sqoop/conf/ranger-sqoop-audit-changes.cfg @@ -89,3 +89,8 @@ xasecure.audit.log4j.async.max.queue.size %XAAUDIT.LOG4J.ASYNC.MA xasecure.audit.log4j.async.max.flush.interval.ms %XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS% mod create-if-not-exists xasecure.audit.destination.log4j %XAAUDIT.LOG4J.DESTINATION.LOG4J% mod create-if-not-exists xasecure.audit.destination.log4j.logger %XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER% mod create-if-not-exists + +#Audit Service Destination +xasecure.audit.destination.auditserver %XAAUDIT.AUDITSERVER.ENABLE% mod create-if-not-exists +xasecure.audit.destination.auditserver.url %XAAUDIT.AUDITSERVER.URL% mod create-if-not-exists +xasecure.audit.destination.auditserver.batch.filespool.dir %XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/plugin-sqoop/pom.xml b/plugin-sqoop/pom.xml index 16436e107..7dd9bfa9a 100644 --- a/plugin-sqoop/pom.xml +++ b/plugin-sqoop/pom.xml @@ -48,12 +48,7 @@ </dependency> <dependency> <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-hdfs</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-solr</artifactId> + <artifactId>ranger-audit-dest-auditserver</artifactId> <version>${project.version}</version> </dependency> <dependency> diff --git a/plugin-sqoop/scripts/install.properties b/plugin-sqoop/scripts/install.properties index 23c9545fc..576d588cd 100644 --- a/plugin-sqoop/scripts/install.properties +++ b/plugin-sqoop/scripts/install.properties @@ -116,6 +116,10 @@ XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE +#Audit Server Provider +XAAUDIT.AUDITSERVER.ENABLE=false +XAAUDIT.AUDITSERVER.URL=http://ranger-audit:7081 +XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR=/var/log/hive/audit/http/spool # End of V3 properties diff --git a/plugin-trino/conf/ranger-trino-audit-changes.cfg b/plugin-trino/conf/ranger-trino-audit-changes.cfg index 1903d98ca..ba06eddc4 100644 --- a/plugin-trino/conf/ranger-trino-audit-changes.cfg +++ b/plugin-trino/conf/ranger-trino-audit-changes.cfg @@ -82,3 +82,8 @@ xasecure.audit.log4j.async.max.queue.size %XAAUDIT.LOG4J.ASYNC.MA xasecure.audit.log4j.async.max.flush.interval.ms %XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS% mod create-if-not-exists xasecure.audit.destination.log4j %XAAUDIT.LOG4J.DESTINATION.LOG4J% mod create-if-not-exists xasecure.audit.destination.log4j.logger %XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER% mod create-if-not-exists + +#Audit Service Destination +xasecure.audit.destination.auditserver %XAAUDIT.AUDITSERVER.ENABLE% mod create-if-not-exists +xasecure.audit.destination.auditserver.url %XAAUDIT.AUDITSERVER.URL% mod create-if-not-exists +xasecure.audit.destination.auditserver.batch.filespool.dir %XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/plugin-trino/pom.xml b/plugin-trino/pom.xml index 3c607bd26..c2b44ea8b 100644 --- a/plugin-trino/pom.xml +++ b/plugin-trino/pom.xml @@ -104,12 +104,7 @@ </dependency> <dependency> <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-hdfs</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-solr</artifactId> + <artifactId>ranger-audit-dest-auditserver</artifactId> <version>${project.version}</version> </dependency> <dependency> diff --git a/plugin-trino/scripts/install.properties b/plugin-trino/scripts/install.properties index 945ff26a4..c9e081f8c 100644 --- a/plugin-trino/scripts/install.properties +++ b/plugin-trino/scripts/install.properties @@ -118,6 +118,10 @@ XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE +#Audit Server Provider +XAAUDIT.AUDITSERVER.ENABLE=false +XAAUDIT.AUDITSERVER.URL=http://ranger-audit:7081 +XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR=/var/log/hive/audit/http/spool # End of V3 properties diff --git a/plugin-yarn/conf/ranger-yarn-audit-changes.cfg b/plugin-yarn/conf/ranger-yarn-audit-changes.cfg index 651372d3e..c81a52746 100644 --- a/plugin-yarn/conf/ranger-yarn-audit-changes.cfg +++ b/plugin-yarn/conf/ranger-yarn-audit-changes.cfg @@ -89,3 +89,8 @@ xasecure.audit.log4j.async.max.queue.size %XAAUDIT.LOG4J.ASYNC.MA xasecure.audit.log4j.async.max.flush.interval.ms %XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS% mod create-if-not-exists xasecure.audit.destination.log4j %XAAUDIT.LOG4J.DESTINATION.LOG4J% mod create-if-not-exists xasecure.audit.destination.log4j.logger %XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER% mod create-if-not-exists + +#Audit Service Destination +xasecure.audit.destination.auditserver %XAAUDIT.AUDITSERVER.ENABLE% mod create-if-not-exists +xasecure.audit.destination.auditserver.url %XAAUDIT.AUDITSERVER.URL% mod create-if-not-exists +xasecure.audit.destination.auditserver.batch.filespool.dir %XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/plugin-yarn/pom.xml b/plugin-yarn/pom.xml index f3bbafdfc..cdc8df9d9 100644 --- a/plugin-yarn/pom.xml +++ b/plugin-yarn/pom.xml @@ -68,12 +68,7 @@ </dependency> <dependency> <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-hdfs</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-solr</artifactId> + <artifactId>ranger-audit-dest-auditserver</artifactId> <version>${project.version}</version> </dependency> <dependency> diff --git a/plugin-yarn/scripts/install.properties b/plugin-yarn/scripts/install.properties index c0be7ea40..1dde78f5c 100644 --- a/plugin-yarn/scripts/install.properties +++ b/plugin-yarn/scripts/install.properties @@ -116,6 +116,10 @@ XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE +#Audit Server Provider +XAAUDIT.AUDITSERVER.ENABLE=false +XAAUDIT.AUDITSERVER.URL=http://ranger-audit:7081 +XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR=/var/log/hive/audit/http/spool # End of V3 properties diff --git a/plugin-yarn/template/configuration.xml b/plugin-yarn/template/configuration.xml new file mode 100644 index 000000000..bdb012589 --- /dev/null +++ b/plugin-yarn/template/configuration.xml @@ -0,0 +1,20 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +<configuration> +</configuration> diff --git a/ranger-examples/plugin-sampleapp/pom.xml b/ranger-examples/plugin-sampleapp/pom.xml index c43776577..e4c563239 100644 --- a/ranger-examples/plugin-sampleapp/pom.xml +++ b/ranger-examples/plugin-sampleapp/pom.xml @@ -90,12 +90,7 @@ </dependency> <dependency> <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-hdfs</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.ranger</groupId> - <artifactId>ranger-audit-dest-solr</artifactId> + <artifactId>ranger-audit-dest-auditserver</artifactId> <version>${project.version}</version> </dependency> <dependency> diff --git a/ranger-kylin-plugin-shim/pom.xml b/ranger-kylin-plugin-shim/pom.xml index 5ec2183e3..6f4328c10 100644 --- a/ranger-kylin-plugin-shim/pom.xml +++ b/ranger-kylin-plugin-shim/pom.xml @@ -59,11 +59,21 @@ <groupId>org.apache.kylin</groupId> <artifactId>kylin-shaded-guava</artifactId> </exclusion> + <exclusion> + <groupId>org.roaringbitmap</groupId> + <artifactId>RoaringBitmap</artifactId> + </exclusion> <exclusion> <groupId>org.slf4j</groupId> <artifactId>*</artifactId> </exclusion> </exclusions> </dependency> + <dependency> + <groupId>org.roaringbitmap</groupId> + <artifactId>RoaringBitmap</artifactId> + <version>0.7.36</version> + <scope>provided</scope> + </dependency> </dependencies> </project> diff --git a/storm-agent/conf/ranger-storm-audit-changes.cfg b/storm-agent/conf/ranger-storm-audit-changes.cfg index 651372d3e..c81a52746 100644 --- a/storm-agent/conf/ranger-storm-audit-changes.cfg +++ b/storm-agent/conf/ranger-storm-audit-changes.cfg @@ -89,3 +89,8 @@ xasecure.audit.log4j.async.max.queue.size %XAAUDIT.LOG4J.ASYNC.MA xasecure.audit.log4j.async.max.flush.interval.ms %XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS% mod create-if-not-exists xasecure.audit.destination.log4j %XAAUDIT.LOG4J.DESTINATION.LOG4J% mod create-if-not-exists xasecure.audit.destination.log4j.logger %XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER% mod create-if-not-exists + +#Audit Service Destination +xasecure.audit.destination.auditserver %XAAUDIT.AUDITSERVER.ENABLE% mod create-if-not-exists +xasecure.audit.destination.auditserver.url %XAAUDIT.AUDITSERVER.URL% mod create-if-not-exists +xasecure.audit.destination.auditserver.batch.filespool.dir %XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/storm-agent/scripts/install.properties b/storm-agent/scripts/install.properties index 80a1ba092..b16d74ff6 100644 --- a/storm-agent/scripts/install.properties +++ b/storm-agent/scripts/install.properties @@ -116,6 +116,10 @@ XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE +#Audit Server Provider +XAAUDIT.AUDITSERVER.ENABLE=false +XAAUDIT.AUDITSERVER.URL=http://ranger-audit:7081 +XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR=/var/log/hive/audit/http/spool # End of V3 properties
