Repository: ambari Updated Branches: refs/heads/trunk bd3082473 -> 176a1c6ba
AMBARI-19147. LogSearch - generate default jks file on startup (oleewere) Change-Id: Ifeb0d1cf6e0a131be969f59c1c6817bb97e67a5a Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/176a1c6b Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/176a1c6b Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/176a1c6b Branch: refs/heads/trunk Commit: 176a1c6ba4187c24d7203a69c5ab257e778956a4 Parents: bd30824 Author: oleewere <[email protected]> Authored: Tue Dec 13 19:49:47 2016 +0100 Committer: oleewere <[email protected]> Committed: Tue Dec 13 20:41:24 2016 +0100 ---------------------------------------------------------------------- .../ambari-logsearch-portal/pom.xml | 5 + .../org/apache/ambari/logsearch/LogSearch.java | 110 ++++++++++++++++-- .../logsearch/common/PropertiesHelper.java | 5 +- .../apache/ambari/logsearch/util/SSLUtil.java | 111 ++++++++++++++++++- .../src/main/resources/default.properties | 3 + .../src/main/resources/log4j.xml | 3 - .../0.5.0/configuration/logsearch-env.xml | 4 +- 7 files changed, 218 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/176a1c6b/ambari-logsearch/ambari-logsearch-portal/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/pom.xml b/ambari-logsearch/ambari-logsearch-portal/pom.xml index 3d218b0..46d543f 100755 --- a/ambari-logsearch/ambari-logsearch-portal/pom.xml +++ b/ambari-logsearch/ambari-logsearch-portal/pom.xml @@ -778,5 +778,10 @@ <artifactId>jersey-bean-validation</artifactId> <version>2.25</version> </dependency> + <dependency> + <groupId>org.bouncycastle</groupId> + <artifactId>bcprov-jdk15on</artifactId> + <version>1.55</version> + </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/ambari/blob/176a1c6b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/LogSearch.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/LogSearch.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/LogSearch.java index 913d25b..614e91e 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/LogSearch.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/LogSearch.java @@ -18,12 +18,17 @@ */ package org.apache.ambari.logsearch; +import java.io.File; import java.io.IOException; import java.net.MalformedURLException; import java.net.ServerSocket; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.security.KeyPair; +import java.security.KeyStore; +import java.security.Security; +import java.security.cert.X509Certificate; import java.util.EnumSet; import org.apache.ambari.logsearch.common.ManageStartEndTime; @@ -31,8 +36,12 @@ import org.apache.ambari.logsearch.common.PropertiesHelper; import org.apache.ambari.logsearch.conf.ApplicationConfig; import org.apache.ambari.logsearch.util.SSLUtil; import org.apache.ambari.logsearch.web.listener.LogSearchSessionListener; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; -import org.apache.log4j.Logger; +import org.apache.tools.ant.Project; +import org.apache.tools.ant.taskdefs.Chmod; +import org.apache.tools.ant.types.FileSet; +import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.eclipse.jetty.server.Connector; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; @@ -49,6 +58,8 @@ import org.eclipse.jetty.util.resource.Resource; import org.eclipse.jetty.util.resource.ResourceCollection; import org.eclipse.jetty.util.ssl.SslContextFactory; import org.eclipse.jetty.webapp.WebAppContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.web.context.ContextLoaderListener; import org.springframework.web.context.request.RequestContextListener; import org.springframework.web.context.support.AnnotationConfigWebApplicationContext; @@ -59,9 +70,11 @@ import javax.servlet.DispatcherType; import static org.apache.ambari.logsearch.common.LogSearchConstants.LOGSEARCH_SESSION_ID; public class LogSearch { - private static final Logger logger = Logger.getLogger(LogSearch.class); + private static final Logger LOG = LoggerFactory.getLogger(LogSearch.class); private static final String LOGSEARCH_PROTOCOL_PROP = "logsearch.protocol"; + private static final String LOGSEARCH_CERT_FOLDER_LOCATION = "logsearch.cert.folder.location"; + private static final String LOGSEARCH_CERT_ALGORITHM = "logsearch.cert.algorithm"; private static final String HTTPS_PROTOCOL = "https"; private static final String HTTP_PROTOCOL = "http"; private static final String HTTPS_PORT = "61889"; @@ -71,6 +84,13 @@ public class LogSearch { private static final String ROOT_CONTEXT = "/"; private static final Integer SESSION_TIMEOUT = 60 * 30; + private static final String LOGSEARCH_CERT_DEFAULT_FOLDER = "/etc/ambari-logsearch-portal/conf/keys"; + private static final String LOGSEARCH_CERT_FILENAME = "logsearch.crt"; + private static final String LOGSEARCH_KEYSTORE_FILENAME = "logsearch.jks"; + private static final String LOGSEARCH_KEYSTORE_PRIVATE_KEY = "logsearch.private.key"; + private static final String LOGSEARCH_KEYSTORE_PUBLIC_KEY = "logsearch.public.key"; + private static final String LOGSEARCH_KEYSTORE_DEFAULT_PASSWORD = "bigdata"; + private static final String LOGSEARCH_CERT_DEFAULT_ALGORITHM = "sha256WithRSAEncryption"; public static void main(String[] argv) { LogSearch logSearch = new LogSearch(); @@ -78,11 +98,12 @@ public class LogSearch { try { logSearch.run(argv); } catch (Throwable e) { - logger.error("Error running logsearch server", e); + LOG.error("Error running logsearch server", e); } } public void run(String[] argv) throws Exception { + loadKeystore(); Server server = buildSever(argv); HandlerList handlers = new HandlerList(); handlers.addHandler(createSwaggerContext()); @@ -91,10 +112,10 @@ public class LogSearch { server.setHandler(handlers); server.start(); - logger + LOG .debug("============================Server Dump======================================="); - logger.debug(server.dump()); - logger + LOG.debug(server.dump()); + LOG .debug("=============================================================================="); server.join(); } @@ -110,7 +131,7 @@ public class LogSearch { } String port = null; if (HTTPS_PROTOCOL.equals(protcolProperty) && SSLUtil.isKeyStoreSpecified()) { - logger.info("Building https server..........."); + LOG.info("Building https server..........."); port = portSpecified ? argv[0] : HTTPS_PORT; checkPort(Integer.parseInt(port)); httpConfiguration.addCustomizer(new SecureRequestCustomizer()); @@ -121,7 +142,7 @@ public class LogSearch { sslConnector.setPort(Integer.parseInt(port)); server.setConnectors(new Connector[] { sslConnector }); } else { - logger.info("Building http server..........."); + LOG.info("Building http server..........."); port = portSpecified ? argv[0] : HTTP_PORT; checkPort(Integer.parseInt(port)); ServerConnector connector = new ServerConnector(server, new HttpConnectionFactory(httpConfiguration)); @@ -129,7 +150,7 @@ public class LogSearch { server.setConnectors(new Connector[] { connector }); } URI logsearchURI = URI.create(String.format("%s://0.0.0.0:%s", protcolProperty, port)); - logger.info("Starting logsearch server URI=" + logsearchURI); + LOG.info("Starting logsearch server URI=" + logsearchURI); return server; } @@ -185,11 +206,11 @@ public class LogSearch { try { return fileCompleteUrl.toURI().normalize(); } catch (URISyntaxException e) { - logger.error(errorMessage, e); + LOG.error(errorMessage, e); System.exit(1); } } else { - logger.error(errorMessage); + LOG.error(errorMessage); System.exit(1); } throw new IllegalStateException(errorMessage); @@ -202,7 +223,7 @@ public class LogSearch { serverSocket = new ServerSocket(port); } catch (IOException ex) { portBusy = true; - logger.error(ex.getLocalizedMessage() + " PORT :" + port); + LOG.error(ex.getLocalizedMessage() + " PORT :" + port); } finally { if (serverSocket != null) { try { @@ -217,4 +238,69 @@ public class LogSearch { } } + /** + * Create keystore with keys and certificate (only if the keystore does not exist or if you have no permissions on the keystore file) + */ + void loadKeystore() { + try { + String certFolder = PropertiesHelper.getProperty(LOGSEARCH_CERT_FOLDER_LOCATION, LOGSEARCH_CERT_DEFAULT_FOLDER); + String certAlgorithm = PropertiesHelper.getProperty(LOGSEARCH_CERT_ALGORITHM, LOGSEARCH_CERT_DEFAULT_ALGORITHM); + String certLocation = String.format("%s/%s", LOGSEARCH_CERT_DEFAULT_FOLDER, LOGSEARCH_CERT_FILENAME); + String keyStoreLocation = StringUtils.isNotEmpty(SSLUtil.getKeyStoreLocation()) ? SSLUtil.getKeyStoreLocation() + : String.format("%s/%s", LOGSEARCH_CERT_DEFAULT_FOLDER, LOGSEARCH_KEYSTORE_FILENAME); + char[] password = StringUtils.isNotEmpty(SSLUtil.getKeyStorePassword()) ? + SSLUtil.getKeyStorePassword().toCharArray() : LOGSEARCH_KEYSTORE_DEFAULT_PASSWORD.toCharArray(); + boolean keyStoreFileExists = new File(keyStoreLocation).exists(); + if (!keyStoreFileExists) { + createDefaultKeyFolder(certFolder); + LOG.warn("Keystore file ('{}') does not exist, creating new one. " + + "If the file exists, make sure you have proper permissions on that.", keyStoreLocation); + if (SSLUtil.isKeyStoreSpecified() && !"JKS".equalsIgnoreCase(SSLUtil.getKeyStoreType())) { + throw new RuntimeException(String.format("Keystore does not exist. Only JKS keystore can be auto generated. (%s)", keyStoreLocation)); + } + LOG.info("SSL keystore is not specified. Generating it with certificate ... (using default format: JKS)"); + Security.addProvider(new BouncyCastleProvider()); + KeyPair keyPair = SSLUtil.createKeyPair("RSA", 2048); + File privateKeyFile = new File(String.format("%s/%s", certFolder, LOGSEARCH_KEYSTORE_PRIVATE_KEY)); + if (!privateKeyFile.exists()) { + FileUtils.writeByteArrayToFile(privateKeyFile, keyPair.getPrivate().getEncoded()); + } + File file = new File(String.format("%s/%s", certFolder, LOGSEARCH_KEYSTORE_PUBLIC_KEY)); + if (!file.exists()) { + FileUtils.writeByteArrayToFile(file, keyPair.getPublic().getEncoded()); + } + X509Certificate cert = SSLUtil.generateCertificate(certLocation, keyPair, certAlgorithm); + KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); + keyStore.load(null, password); + SSLUtil.setKeyAndCertInKeystore(cert, keyPair, keyStore, keyStoreLocation, password); + setPermissionOnCertFolder(certFolder); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void createDefaultKeyFolder(String certFolder) { + File keyFolderDirectory = new File(certFolder); + if (!keyFolderDirectory.exists()) { + LOG.info("Default key dir does not exist ({}). Creating ...", certFolder); + boolean mkDirSuccess = keyFolderDirectory.mkdirs(); + if (!mkDirSuccess) { + String errorMessage = String.format("Could not create directory %s", certFolder); + LOG.error(errorMessage); + throw new RuntimeException(errorMessage); + } + } + } + + private void setPermissionOnCertFolder(String certFolder) { + Chmod chmod = new Chmod(); + chmod.setProject(new Project()); + FileSet fileSet = new FileSet(); + fileSet.setDir(new File(certFolder)); + fileSet.setIncludes("**"); + chmod.addFileset(fileSet); + chmod.setPerm("640"); + chmod.execute(); + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/176a1c6b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/PropertiesHelper.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/PropertiesHelper.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/PropertiesHelper.java index 257ae3c..73a43ad 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/PropertiesHelper.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/PropertiesHelper.java @@ -28,13 +28,14 @@ import java.util.Map; import java.util.Properties; import java.util.Set; -import org.apache.log4j.Logger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.BeansException; import org.springframework.beans.factory.config.ConfigurableListableBeanFactory; import org.springframework.beans.factory.config.PropertyPlaceholderConfigurer; public class PropertiesHelper extends PropertyPlaceholderConfigurer { - private static final Logger logger = Logger.getLogger(PropertiesHelper.class); + private static final Logger logger = LoggerFactory.getLogger(PropertiesHelper.class); private static final String LOGSEARCH_PROP_FILE="logsearch.properties"; http://git-wip-us.apache.org/repos/asf/ambari/blob/176a1c6b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SSLUtil.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SSLUtil.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SSLUtil.java index 206f793..7a93305 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SSLUtil.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SSLUtil.java @@ -21,12 +21,37 @@ package org.apache.ambari.logsearch.util; import javax.net.ssl.SSLContext; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; -import org.apache.log4j.Logger; +import org.bouncycastle.jce.X509Principal; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.x509.X509V3CertificateGenerator; import org.eclipse.jetty.util.ssl.SslContextFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.math.BigInteger; +import java.net.InetAddress; +import java.security.InvalidKeyException; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.KeyStore; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; +import java.security.SecureRandom; +import java.security.Security; +import java.security.SignatureException; +import java.security.cert.Certificate; +import java.security.cert.CertificateEncodingException; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; +import java.util.Date; public class SSLUtil { - private static final Logger logger = Logger.getLogger(SSLUtil.class); + private static final Logger LOG = LoggerFactory.getLogger(SSLUtil.class); private static final String KEYSTORE_LOCATION_ARG = "javax.net.ssl.keyStore"; private static final String KEYSTORE_PASSWORD_ARG = "javax.net.ssl.keyStorePassword"; @@ -94,14 +119,92 @@ public class SSLUtil { sslContextFactory.start(); return sslContextFactory.getSslContext(); } catch (Exception e) { - logger.error("Could not create SSL Context", e); + LOG.error("Could not create SSL Context", e); return null; } finally { try { sslContextFactory.stop(); } catch (Exception e) { - logger.error("Could not stop sslContextFactory", e); + LOG.error("Could not stop sslContextFactory", e); + } + } + } + + /** + * Put private key into in-memory keystore and write it to a file (JKS file) + */ + public static void setKeyAndCertInKeystore(X509Certificate cert, KeyPair keyPair, KeyStore keyStore, String keyStoreLocation, char[] password) + throws Exception { + Certificate[] certChain = new Certificate[1]; + certChain[0] = cert; + try (FileOutputStream fos = new FileOutputStream(keyStoreLocation)) { + keyStore.setKeyEntry("logsearch.alias", keyPair.getPrivate(), password, certChain); + keyStore.store(fos, password); + } catch (Exception e) { + LOG.error("Could not write certificate to Keystore"); + throw e; + } + } + + /** + * Create in-memory keypair with bouncy castle + */ + public static KeyPair createKeyPair(String encryptionType, int byteCount) + throws NoSuchProviderException, NoSuchAlgorithmException { + Security.addProvider(new BouncyCastleProvider()); + KeyPairGenerator keyPairGenerator = createKeyPairGenerator(encryptionType, byteCount); + return keyPairGenerator.genKeyPair(); + } + + /** + * Generate X509 certificate if it does not exist + */ + public static X509Certificate generateCertificate(String certificateLocation, KeyPair keyPair, String algorithm) throws Exception { + try { + File certFile = new File(certificateLocation); + if (certFile.exists()) { + LOG.info("Certificate file exists ({}), skip the generation.", certificateLocation); + return getCertFile(certificateLocation); + } else { + Security.addProvider(new BouncyCastleProvider()); + X509Certificate cert = SSLUtil.createCert(keyPair, algorithm, InetAddress.getLocalHost().getCanonicalHostName()); + FileUtils.writeByteArrayToFile(certFile, cert.getEncoded()); + return cert; } + } catch (Exception e) { + LOG.error("Could not create certificate."); + throw e; + } + } + + private static X509Certificate getCertFile(String location) throws Exception { + try (FileInputStream fos = new FileInputStream(location)) { + CertificateFactory factory = CertificateFactory.getInstance("X.509"); + return (X509Certificate) factory.generateCertificate(fos); + } catch (Exception e) { + LOG.error("Cannot read cert file. ('{}')", location); + throw e; } } + + private static X509Certificate createCert(KeyPair keyPair, String signatureAlgoritm, String domainName) + throws CertificateEncodingException, NoSuchAlgorithmException, InvalidKeyException, SignatureException { + X509V3CertificateGenerator v3CertGen = new X509V3CertificateGenerator(); + v3CertGen.setSerialNumber(BigInteger.valueOf(Math.abs(new SecureRandom().nextInt()))); + v3CertGen.setIssuerDN(new X509Principal("CN=" + domainName + ", OU=None, O=None L=None, C=None")); + v3CertGen.setNotBefore(new Date(System.currentTimeMillis() - 1000L * 60 * 60 * 24 * 30)); + v3CertGen.setNotAfter(new Date(System.currentTimeMillis() + (1000L * 60 * 60 * 24 * 365*10))); + v3CertGen.setSubjectDN(new X509Principal("CN=" + domainName + ", OU=None, O=None L=None, C=None")); + v3CertGen.setPublicKey(keyPair.getPublic()); + v3CertGen.setSignatureAlgorithm(signatureAlgoritm); + return v3CertGen.generate(keyPair.getPrivate()); + } + + private static KeyPairGenerator createKeyPairGenerator(String algorithmIdentifier, int bitCount) + throws NoSuchProviderException, NoSuchAlgorithmException { + KeyPairGenerator kpg = KeyPairGenerator.getInstance(algorithmIdentifier, BouncyCastleProvider.PROVIDER_NAME); + kpg.initialize(bitCount); + return kpg; + } + } http://git-wip-us.apache.org/repos/asf/ambari/blob/176a1c6b/ambari-logsearch/ambari-logsearch-portal/src/main/resources/default.properties ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/resources/default.properties b/ambari-logsearch/ambari-logsearch-portal/src/main/resources/default.properties index c98a482..cbfe157 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/resources/default.properties +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/resources/default.properties @@ -24,3 +24,6 @@ logsearch.logfeeder.include.default.level=FATAL,ERROR,WARN,INFO,DEBUG,TRACE #login config logsearch.login.credentials.file=user_pass.json logsearch.login.ldap.config=logsearch-admin-site.xml + +logsearch.cert.folder.location=/etc/ambari-logsearch-portal/conf/keys +logsearch.cert.algorithm=sha256WithRSAEncryption http://git-wip-us.apache.org/repos/asf/ambari/blob/176a1c6b/ambari-logsearch/ambari-logsearch-portal/src/main/resources/log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/resources/log4j.xml b/ambari-logsearch/ambari-logsearch-portal/src/main/resources/log4j.xml index 60b09cd..547841a 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/resources/log4j.xml +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/resources/log4j.xml @@ -94,20 +94,17 @@ <logger name="org.apache.ambari.logsearch.audit" additivity="true"> - <priority value="info" /> <!-- <appender-ref ref="audit_rolling_file" />--> <appender-ref ref="audit_rolling_file_json" /> </logger> <logger name="org.apache.ambari.logsearch.performance" additivity="false"> - <priority value="info" /> <!-- <appender-ref ref="performance_analyzer" />--> <appender-ref ref="performance_analyzer_json" /> </logger> <logger name="org.apache.ambari.logsearch" additivity="false"> - <priority value="info" /> <!-- <appender-ref ref="console" /> --> <!-- <appender-ref ref="rolling_file" />--> <appender-ref ref="rolling_file_json" /> http://git-wip-us.apache.org/repos/asf/ambari/blob/176a1c6b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-env.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-env.xml index 2f13710..8691d35 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-env.xml +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-env.xml @@ -194,7 +194,7 @@ </property> <property> <name>logsearch_truststore_location</name> - <value>/etc/security/serverKeys/logsearch.trustStore.jks</value> + <value>/etc/ambari-logsearch-portal/conf/keys/logsearch.jks</value> <display-name>Log Search trust store location</display-name> <description>Location of the trust store file.</description> <on-ambari-upgrade add="true"/> @@ -219,7 +219,7 @@ </property> <property> <name>logsearch_keystore_location</name> - <value>/etc/security/serverKeys/logsearch.keyStore.jks</value> + <value>/etc/ambari-logsearch-portal/conf/keys/logsearch.jks</value> <display-name>Log Search key store location</display-name> <description>Location of the key store file.</description> <on-ambari-upgrade add="true"/>
