pradeepagrawal8184 commented on code in PR #1103: URL: https://github.com/apache/ranger/pull/1103#discussion_r3814883651
########## ugsync/src/main/java/org/apache/ranger/metrics/MetricCacheUtil.java: ########## @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.metrics; + +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; + +public class MetricCacheUtil { + public static final String ADD_USER_COUNT_SUCCESS = "AddUserCountSUCCESS"; + public static final String ADD_USER_COUNT_FAIL = "AddUserCountFAIL"; + public static final String GROUP_USER_COUNT_SUCCESS = "GroupUserCountSUCCESS"; + public static final String GROUP_USER_COUNT_FAIL = "GroupUserCountFAIL"; + public static final String ADD_GROUP_COUNT_SUCCESS = "AddGroupCountSUCCESS"; + public static final String ADD_GROUP_COUNT_FAIL = "AddGroupCountFAIL"; + public static final String DELETE_USER_COUNT_SUCCESS = "DeleteUserCountSUCCESS"; + public static final String DELETE_USER_COUNT_FAIL = "DeleteUserCountFAIL"; + public static final String DELETE_GROUP_COUNT_SUCCESS = "DeleteGroupCountSUCCESS"; + public static final String DELETE_GROUP_COUNT_FAIL = "DeleteGroupCountFAIL"; + public static final String AUDIT_COUNT_SUCCESS = "AuditCountSUCCESS"; + public static final String AUDIT_COUNT_FAIL = "AuditCountFAIL"; + public static final String NO_OF_CACHED_USERS = "CountUSER"; + public static final String NO_OF_CACHED_GROUPS = "CountGROUP"; + public static final String NO_OF_CACHED_GROUPS_USERS = "CountGROUPUSER"; + private static MetricCacheUtil metricCacheUtil; + private Map<String, Long> apiMetrics = new ConcurrentHashMap<>(); + private Map<String, Long> cacheMetrics = new ConcurrentHashMap<>(); + private Map<String, Long> sourceMetrics = new ConcurrentHashMap<>(); + + private MetricCacheUtil() { + } + + public static MetricCacheUtil getInstance() { + if (Objects.isNull(metricCacheUtil)) { + synchronized (MetricCacheUtil.class) { + if (Objects.isNull(metricCacheUtil)) { + metricCacheUtil = new MetricCacheUtil(); + } + } + } + return metricCacheUtil; + } + + public void incrementMetric(MetricType metricType, String metricName, Long newValue) { + Map<String, Long> metric = getMetric(metricType); + + if (Objects.nonNull(metric)) { + if (metricType.equals(MetricType.CACHE) || metricType.equals(MetricType.SYNCSOURCE)) { + metric.put(metricName, newValue); + } else { + metric.merge(metricName, newValue, Long::sum); + } + } + } + + public Map<String, Long> getMetric(MetricType metricType) { Review Comment: Return defensive copies from getMetric() ########## unixauthservice/scripts/ranger-usersync-services.sh: ########## @@ -15,6 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -x Review Comment: do we need this line ? ########## ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java: ########## @@ -1375,28 +1393,41 @@ private void getUserGroupAuditInfo(UgsyncAuditInfo userInfo) { checkStatus(); - String response = null; - Response clientRes = null; + String response = null; if (isRangerCookieEnabled) { response = cookieBasedUploadEntity(userInfo, PM_AUDIT_INFO_URI); } else { try { - clientRes = ldapUgSyncClient.post(PM_AUDIT_INFO_URI, null, userInfo); + Response clientRes = ldapUgSyncClient.post(PM_AUDIT_INFO_URI, null, userInfo); if (clientRes != null) { response = clientRes.readEntity(String.class); } } catch (Throwable t) { LOG.error("Failed to get response, Error is : ", t); + metricCacheUtil.incrementMetric(MetricCacheUtil.MetricType.API, MetricCacheUtil.AUDIT_COUNT_FAIL, 1L); + LOG.debug("<== PolicyMgrUserGroupBuilder.getUserGroupAuditInfo()"); + return; } } LOG.debug("REST response from {} : {}", PM_AUDIT_INFO_URI, response); - JsonUtils.jsonToObject(response, UgsyncAuditInfo.class); + if (StringUtils.isNotEmpty(response)) { + try { + JsonUtils.jsonToObject(response, UgsyncAuditInfo.class); + metricCacheUtil.incrementMetric(MetricCacheUtil.MetricType.API, MetricCacheUtil.AUDIT_COUNT_SUCCESS, 1L); + LOG.debug("AuditInfo Creation successful "); + } catch (Exception e) { + LOG.error("Failed to parse audit info response", e); + metricCacheUtil.incrementMetric(MetricCacheUtil.MetricType.API, MetricCacheUtil.AUDIT_COUNT_FAIL, 1L); + } + } else { + LOG.error("Failed to post audit info - empty response from {}", PM_AUDIT_INFO_URI); + metricCacheUtil.incrementMetric(MetricCacheUtil.MetricType.API, MetricCacheUtil.AUDIT_COUNT_FAIL, 1L); Review Comment: in the line 1409 this is already done: On the cookie path, checkFailureApis() in tryUploadEntityWithCookie / tryUploadEntityWithCred already increments AUDIT_COUNT_FAIL. The empty-response branch was incrementing it again. replace this else block with below: `} else { LOG.error("Failed to post audit info - empty response from {}", PM_AUDIT_INFO_URI); if (!isRangerCookieEnabled) { metricCacheUtil.incrementMetric(MetricCacheUtil.MetricType.API, MetricCacheUtil.AUDIT_COUNT_FAIL, 1L); } }` ########## unixauthservice/src/main/java/org/apache/ranger/authentication/server/EmbeddedServer.java: ########## @@ -0,0 +1,833 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.authentication.server; + +import org.apache.catalina.Context; +import org.apache.catalina.LifecycleException; +import org.apache.catalina.WebResourceRoot; +import org.apache.catalina.connector.Connector; +import org.apache.catalina.core.StandardContext; +import org.apache.catalina.startup.Tomcat; +import org.apache.catalina.valves.AccessLogValve; +import org.apache.catalina.webresources.StandardRoot; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.SecureClientLogin; +import org.apache.hadoop.security.alias.BouncyCastleFipsKeyStoreProvider; +import org.apache.hadoop.security.alias.CredentialProvider; +import org.apache.hadoop.security.alias.CredentialProviderFactory; +import org.apache.hadoop.security.alias.JavaKeyStoreProvider; +import org.apache.hadoop.security.alias.LocalBouncyCastleFipsKeyStoreProvider; +import org.apache.ranger.authorization.hadoop.utils.RangerCredentialProvider; +import org.apache.ranger.unixusersync.config.UserGroupSyncConfig; +import org.apache.tomcat.util.net.SSLHostConfig; +import org.apache.tomcat.util.scan.StandardJarScanner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.KeyManager; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import javax.security.auth.Subject; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.security.KeyManagementException; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.PrivilegedAction; +import java.security.SecureRandom; +import java.security.UnrecoverableKeyException; +import java.security.cert.CertificateException; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +public class EmbeddedServer { + private static final Logger LOG = LoggerFactory.getLogger(EmbeddedServer.class); + + private static final String RANGER_SSL_CONTEXT_ALGO_TYPE = "TLS"; + private static final String DEFAULT_SSL_PROTOCOL = "TLS"; + private static final String RANGER_SSL_KEYMANAGER_ALGO_TYPE = KeyManagerFactory.getDefaultAlgorithm(); + private static final String RANGER_SSL_TRUSTMANAGER_ALGO_TYPE = TrustManagerFactory.getDefaultAlgorithm(); + public static final String RANGER_KERBEROS_KEYTAB = "kerberos.keytab"; + public static final String RANGER_KERBEROS_PRINCIPAL = "kerberos.principal"; + public static final String HADOOP_KERBEROS_NAME_RULES = "hadoop.security.auth_to_local"; + public static final String HADOOP_AUTHENTICATION_TYPE = "hadoop.security.authentication"; + public static final String HADOOP_AUTH_TYPE_KERBEROS = "kerberos"; + private static final String ACCESS_LOG_PREFIX = "accesslog.prefix"; + private static final String ACCESS_LOG_DATE_FORMAT = "accesslog.dateformat"; + private static final String ACCESS_LOG_PATTERN = "accesslog.pattern"; + private static final String ACCESS_LOG_ROTATE_MAX_DAYS = "accesslog.rotate.max.days"; + private static final int DEFAULT_HTTP_PORT = 8280; + private static final int DEFAULT_HTTPS_PORT = 8283; + private static final int DEFAULT_SHUTDOWN_PORT = 8285; + private static final String DEFAULT_SHUTDOWN_COMMAND = "SHUTDOWN"; + private static final String DEFAULT_WEBAPPS_ROOT_FOLDER = "webapps"; + private static final String DEFAULT_ENABLED_PROTOCOLS = "TLSv1.2"; + public static final String DEFAULT_NAME_RULE = "DEFAULT"; + public static final String KEYSTORE_FILE_TYPE_DEFAULT = KeyStore.getDefaultType(); + public static final String TRUSTSTORE_FILE_TYPE_DEFAULT = KeyStore.getDefaultType(); + + private final Configuration configuration; + private final String appName; + private final String configPrefix; + + private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); + private Properties prop = new Properties(); + + public static void main(String[] args) { + String appName = "ranger-usersync"; + String configPrefix = "ranger.usersync."; + + new EmbeddedServer(appName, configPrefix).start(); + } + + public EmbeddedServer(String appName, String configPrefix) { + LOG.info("==> EmbeddedServer(appName={}, configPrefix={})", appName, configPrefix); + + this.configuration = config.getConfig(); + this.appName = appName; + this.configPrefix = configPrefix; + prop = config.getProperties(); + + LOG.info("<== EmbeddedServer(appName={}, configPrefix={}", appName, configPrefix); + } + + @SuppressWarnings("deprecation") + public void start() { + LOG.info("==> EmbeddedServer.start(appName={})", appName); + + SSLContext sslContext = getSSLContext(); + + if (sslContext != null) { + SSLContext.setDefault(sslContext); + } + + final Tomcat server = new Tomcat(); + + String logDir = getProperty("log.dir"); + String hostName = getProperty("service.host"); + int serverPort = getIntProperty("service.http.port", DEFAULT_HTTP_PORT); + int sslPort = getIntProperty("service.https.port", DEFAULT_HTTPS_PORT); + int shutdownPort = getIntProperty("service.shutdown.port", DEFAULT_SHUTDOWN_PORT); + String shutdownCommand = getProperty("service.shutdown.command", DEFAULT_SHUTDOWN_COMMAND); + boolean isHttpsEnabled = getBooleanProperty("service.https.attrib.ssl.enabled", false); + boolean ajpEnabled = getBooleanProperty("ajp.enabled", false); + + server.setHostname(hostName); + server.setPort(serverPort); + server.getServer().setPort(shutdownPort); + server.getServer().setShutdown(shutdownCommand); + if (ajpEnabled) { + Connector ajpConnector = new Connector("org.apache.coyote.ajp.AjpNioProtocol"); + + ajpConnector.setPort(serverPort); + ajpConnector.setProperty("protocol", "AJP/1.3"); + + server.getService().addConnector(ajpConnector); + + // Making this as a default connector + server.setConnector(ajpConnector); + + LOG.info("Created AJP Connector"); + } else if (isHttpsEnabled && sslPort > 0) { + String clientAuth = getProperty("service.https.attrib.clientAuth", "false"); + String providerPath = getProperty("credential.provider.path"); + String keyAlias = getProperty("service.https.attrib.keystore.credential.alias", "keyStoreCredentialAlias"); + String keystorePass = null; + String enabledProtocols = getProperty("service.https.attrib.ssl.enabled.protocols", DEFAULT_ENABLED_PROTOCOLS); + String ciphers = getProperty("tomcat.ciphers"); + + if (StringUtils.equalsIgnoreCase(clientAuth, "false")) { + clientAuth = getProperty("service.https.attrib.client.auth", "want"); + } + + String keystoreFileType = getKeystoreFileType(); + String truststoreType = getTruststoreFileType(); + + if (providerPath != null && keyAlias != null) { + keystorePass = getDecryptedString(providerPath.trim(), keyAlias.trim(), keystoreFileType); + if (StringUtils.isBlank(keystorePass) || StringUtils.equalsIgnoreCase(keystorePass.trim(), "none")) { + keystorePass = getProperty("service.https.attrib.keystore.pass"); + } + } + + String keystoreFile = getKeystoreFile(); + String sslKeystoreKeyAlias = getProperty("service.https.attrib.keystore.keyalias", ""); + String validationError = validateHttpsKeystore(keystoreFile, keystorePass, sslKeystoreKeyAlias, keystoreFileType); + + if (validationError != null) { + LOG.error("HTTPS configuration validation failed: {} The HTTPS connector may not bind to port {} " + + "and the Usersync service may be unavailable.", validationError, sslPort); + } + + Connector ssl = new Connector(); + ssl.setPort(sslPort); + ssl.setSecure(true); + ssl.setScheme("https"); + ssl.setAttribute("SSLEnabled", "true"); + ssl.setAttribute("sslProtocol", getProperty("service.https.attrib.ssl.protocol", DEFAULT_SSL_PROTOCOL)); + ssl.setAttribute("keystoreType", keystoreFileType); + ssl.setAttribute("truststoreType", truststoreType); + ssl.setAttribute("clientAuth", clientAuth); + if (StringUtils.isNotBlank(sslKeystoreKeyAlias)) { + ssl.setAttribute("keyAlias", sslKeystoreKeyAlias); + } + ssl.setAttribute("keystorePass", keystorePass); + ssl.setAttribute("keystoreFile", keystoreFile); + ssl.setAttribute("sslEnabledProtocols", enabledProtocols); + + if (StringUtils.isNotBlank(ciphers)) { + ssl.setAttribute("ciphers", ciphers); + SSLHostConfig[] configs = ssl.findSslHostConfigs(); + + if (configs != null) { + for (SSLHostConfig hostConfig : configs) { + if (hostConfig != null) { + hostConfig.setCipherSuites(ciphers); + } + } + } + } + + server.getService().addConnector(ssl); + + // + // Making this as a default connector + // + server.setConnector(ssl); + } + updateHttpConnectorAttribConfig(server); + + File logDirectory = new File(logDir); + + if (!logDirectory.exists()) { + logDirectory.mkdirs(); + } + + String logPattern = getProperty(ACCESS_LOG_PATTERN, "%h %l %u %t \"%r\" %s %b"); + + AccessLogValve valve = new AccessLogValve(); + + valve.setRotatable(true); + valve.setAsyncSupported(true); + valve.setBuffered(false); + valve.setEnabled(true); + valve.setPrefix(getProperty(ACCESS_LOG_PREFIX, "access_log-" + hostName + "-")); + valve.setFileDateFormat(getProperty(ACCESS_LOG_DATE_FORMAT, "yyyy-MM-dd.HH")); + valve.setDirectory(logDirectory.getAbsolutePath()); + valve.setSuffix(".log"); + valve.setPattern(logPattern); + valve.setMaxDays(getIntProperty(ACCESS_LOG_ROTATE_MAX_DAYS, 15)); + + server.getHost().getPipeline().addValve(valve); + try { + String webappDir = getProperty("webapp.dir"); + LOG.info("==> EmbeddedServer.start(webappDir={})", webappDir); + if (StringUtils.isBlank(webappDir)) { + LOG.error("Tomcat Server failed to start: {}.webapp.dir is not set", configPrefix); + + System.exit(1); + } + + String webContextName = getProperty("contextName", ""); + + if (StringUtils.isBlank(webContextName)) { + webContextName = ""; + } else if (!webContextName.startsWith("/")) { + LOG.info("Context Name [{}] is being loaded as [ /{}]", webContextName, webContextName); + + webContextName = "/" + webContextName; + } + + File wad = new File(webappDir); + + if (wad.isDirectory()) { + LOG.info("Webapp dir={}, webAppName={}", webappDir, webContextName); + } else if (wad.isFile()) { + File webAppDir = new File(DEFAULT_WEBAPPS_ROOT_FOLDER); + + if (!webAppDir.exists()) { + webAppDir.mkdirs(); + } + + LOG.info("Webapp file={}, webAppName={}", webappDir, webContextName); + } + + LOG.info("Adding webapp [{}] = path [{}] .....", webContextName, webappDir); + + Context webappCtx = server.addWebapp(webContextName, new File(webappDir).getAbsolutePath()); + if (webappCtx instanceof StandardContext) { + boolean allowLinking = getBooleanProperty("allow.linking", true); + StandardContext standardContext = (StandardContext) webappCtx; + String workDirPath = getProperty("tomcat.work.dir", ""); + if (!workDirPath.isEmpty() && new File(workDirPath).exists()) { + standardContext.setWorkDir(workDirPath); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Skipping to set tomcat server work directory, '" + workDirPath + + "', as it is blank or directory does not exist."); + } + } + WebResourceRoot resRoot = new StandardRoot(standardContext); + webappCtx.setResources(resRoot); + webappCtx.getResources().setAllowLinking(allowLinking); + StandardJarScanner scanner = new StandardJarScanner(); + scanner.setScanManifest(false); + webappCtx.setJarScanner(scanner); + if (LOG.isDebugEnabled()) { + LOG.debug("Tomcat Configuration - allowLinking=[{}]", allowLinking); + } + } else { + LOG.error("Tomcat Context [{}] is either NULL OR it's NOT instanceof StandardContext", webappCtx); + } + + webappCtx.init(); + + LOG.info("Finished init of webapp [{}] = path [{}].", webContextName + webappDir); Review Comment: i think you need replace + with comma ########## unixauthservice/src/main/java/org/apache/ranger/authentication/server/RangerUserSyncServer.java: ########## @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.authentication.server; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class RangerUserSyncServer { + private static final Logger LOG = LoggerFactory.getLogger(RangerUserSyncServer.class); + + private RangerUserSyncServer() { + } + + public static void main(String[] args) { + LOG.info("==>> RangerUserSyncServer.main()"); + try { + EmbeddedServer server = new EmbeddedServer("ranger-usersync", "ranger.usersync."); + server.start(); + } catch (Throwable e) { + LOG.error("Failed to initialize embedded server due to: ", e); Review Comment: check if we need System.exit(1); after line 37 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
