This is an automated email from the ASF dual-hosted git repository. elserj pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/phoenix-queryserver.git
The following commit(s) were added to refs/heads/master by this push: new b25ab3e PHOENIX-5778 Remove the dependency of KeyStoreTestUtil b25ab3e is described below commit b25ab3e2c75229a1a75d94f5e1cfe6d5cfa91282 Author: Guanghao Zhang <zg...@apache.org> AuthorDate: Fri Jul 3 20:55:44 2020 -0400 PHOENIX-5778 Remove the dependency of KeyStoreTestUtil Co-authored-by: Josh Elser <els...@apache.org> Closes #22, closes #41 --- phoenix-queryserver-it/pom.xml | 7 +- .../phoenix/end2end/QueryServerEnvironment.java | 5 +- .../end2end/SecureQueryServerPhoenixDBIT.java | 5 +- .../java/org/apache/phoenix/end2end/TlsUtil.java | 268 ++++++++++++++++++++- pom.xml | 6 + 5 files changed, 275 insertions(+), 16 deletions(-) diff --git a/phoenix-queryserver-it/pom.xml b/phoenix-queryserver-it/pom.xml index 6979f79..961d65b 100644 --- a/phoenix-queryserver-it/pom.xml +++ b/phoenix-queryserver-it/pom.xml @@ -207,6 +207,11 @@ <artifactId>hadoop-common</artifactId> <type>test-jar</type> <scope>test</scope> - </dependency> + </dependency> + <dependency> + <groupId>bouncycastle</groupId> + <artifactId>bcprov-jdk15</artifactId> + <scope>test</scope> + </dependency> </dependencies> </project> diff --git a/phoenix-queryserver-it/src/it/java/org/apache/phoenix/end2end/QueryServerEnvironment.java b/phoenix-queryserver-it/src/it/java/org/apache/phoenix/end2end/QueryServerEnvironment.java index 0d1c55f..7bb6c9e 100644 --- a/phoenix-queryserver-it/src/it/java/org/apache/phoenix/end2end/QueryServerEnvironment.java +++ b/phoenix-queryserver-it/src/it/java/org/apache/phoenix/end2end/QueryServerEnvironment.java @@ -33,7 +33,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.LocalHBaseCluster; -import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil; import org.apache.hadoop.hbase.security.HBaseKerberosUtils; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hdfs.DFSConfigKeys; @@ -179,8 +178,8 @@ public class QueryServerEnvironment { // Generate SSL certs File keystoresDir = new File(UTIL.getDataTestDir("keystore").toUri().getPath()); keystoresDir.mkdirs(); - String sslConfDir = KeyStoreTestUtil.getClasspathDir(QueryServerEnvironment.class); - KeyStoreTestUtil.setupSSLConfig(keystoresDir.getAbsolutePath(), sslConfDir, conf, false); + String sslConfDir = TlsUtil.getClasspathDir(QueryServerEnvironment.class); + TlsUtil.setupSSLConfig(keystoresDir.getAbsolutePath(), sslConfDir, conf, false); // Magic flag to tell hdfs to not fail on using ports above 1024 conf.setBoolean("ignore.secure.ports.for.testing", true); diff --git a/phoenix-queryserver-it/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java b/phoenix-queryserver-it/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java index 2be195a..bab190c 100644 --- a/phoenix-queryserver-it/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java +++ b/phoenix-queryserver-it/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java @@ -50,7 +50,6 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.LocalHBaseCluster; import org.apache.hadoop.hbase.client.TestHCM.SleepAndFailFirstTime; import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; -import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil; import org.apache.hadoop.hbase.security.HBaseKerberosUtils; import org.apache.hadoop.hbase.security.token.TokenProvider; import org.apache.hadoop.hbase.util.FSUtils; @@ -160,8 +159,8 @@ public class SecureQueryServerPhoenixDBIT { // Generate SSL certs File keystoresDir = new File(UTIL.getDataTestDir("keystore").toUri().getPath()); keystoresDir.mkdirs(); - String sslConfDir = KeyStoreTestUtil.getClasspathDir(SecureQueryServerPhoenixDBIT.class); - KeyStoreTestUtil.setupSSLConfig(keystoresDir.getAbsolutePath(), sslConfDir, conf, false); + String sslConfDir = TlsUtil.getClasspathDir(SecureQueryServerPhoenixDBIT.class); + TlsUtil.setupSSLConfig(keystoresDir.getAbsolutePath(), sslConfDir, conf, false); // Magic flag to tell hdfs to not fail on using ports above 1024 conf.setBoolean("ignore.secure.ports.for.testing", true); diff --git a/phoenix-queryserver-it/src/it/java/org/apache/phoenix/end2end/TlsUtil.java b/phoenix-queryserver-it/src/it/java/org/apache/phoenix/end2end/TlsUtil.java index de757b8..ef4fe9c 100644 --- a/phoenix-queryserver-it/src/it/java/org/apache/phoenix/end2end/TlsUtil.java +++ b/phoenix-queryserver-it/src/it/java/org/apache/phoenix/end2end/TlsUtil.java @@ -11,15 +11,38 @@ package org.apache.phoenix.end2end; import java.io.File; +import java.io.FileOutputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.Writer; +import java.math.BigInteger; +import java.net.URL; +import java.security.GeneralSecurityException; +import java.security.InvalidKeyException; +import java.security.Key; import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.KeyStore; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; +import java.security.SecureRandom; +import java.security.SignatureException; +import java.security.cert.Certificate; +import java.security.cert.CertificateEncodingException; import java.security.cert.X509Certificate; +import java.util.Date; import java.util.HashMap; import java.util.Map; +import javax.security.auth.x500.X500Principal; -import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory; +import org.apache.hadoop.security.ssl.SSLFactory; +import org.bouncycastle.x509.X509V1CertificateGenerator; + public class TlsUtil { protected static final String KEYSTORE_PASSWORD = "avaticasecret"; @@ -54,11 +77,11 @@ public class TlsUtil { setupTls(); } catch (Exception e) { LOG.error("could not set upt TLS for HTTPS tests", e); + throw new RuntimeException(e); } } /** - * This is simplified from org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil.setupSSLConfig() * Performs setup of SSL configuration in preparation for testing an SSLFactory. This includes * keys, certs, keystores, truststores. */ @@ -76,18 +99,245 @@ public class TlsUtil { // may not exist } - KeyPair sKP = KeyStoreTestUtil.generateKeyPair("RSA"); + KeyPair sKP = generateKeyPair("RSA"); X509Certificate sCert = - KeyStoreTestUtil.generateCertificate("CN=localhost, O=server", sKP, 30, - "SHA1withRSA"); - KeyStoreTestUtil.createKeyStore(KEYSTORE.getCanonicalPath(), KEYSTORE_PASSWORD, "server", - sKP.getPrivate(), sCert); + generateCertificate("CN=localhost, O=server", sKP, 30, "SHA1withRSA"); + createKeyStore(KEYSTORE.getCanonicalPath(), KEYSTORE_PASSWORD, "server", sKP.getPrivate(), + sCert); Map<String, X509Certificate> certs = new HashMap<String, X509Certificate>(); certs.put("server", sCert); - KeyStoreTestUtil.createTrustStore(TRUSTSTORE.getCanonicalPath(), TRUSTSTORE_PASSWORD, - certs); + createTrustStore(TRUSTSTORE.getCanonicalPath(), TRUSTSTORE_PASSWORD, certs); + } + + /** + * Create a self-signed X.509 Certificate. + * + * @param dn the X.509 Distinguished Name, eg "CN=Test, L=London, C=GB" + * @param pair the KeyPair + * @param days how many days from now the Certificate is valid for + * @param algorithm the signing algorithm, eg "SHA1withRSA" + * @return the self-signed certificate + */ + private static X509Certificate generateCertificate(String dn, KeyPair pair, int days, + String algorithm) + throws CertificateEncodingException, InvalidKeyException, IllegalStateException, + NoSuchProviderException, NoSuchAlgorithmException, SignatureException { + Date from = new Date(); + Date to = new Date(from.getTime() + days * 86400000L); + BigInteger sn = new BigInteger(64, new SecureRandom()); + KeyPair keyPair = pair; + X509V1CertificateGenerator certGen = new X509V1CertificateGenerator(); + X500Principal dnName = new X500Principal(dn); + + certGen.setSerialNumber(sn); + certGen.setIssuerDN(dnName); + certGen.setNotBefore(from); + certGen.setNotAfter(to); + certGen.setSubjectDN(dnName); + certGen.setPublicKey(keyPair.getPublic()); + certGen.setSignatureAlgorithm(algorithm); + X509Certificate cert = certGen.generate(pair.getPrivate()); + return cert; + } + + public static String getClasspathDir(Class<?> klass) throws Exception { + String file = klass.getName(); + file = file.replace('.', '/') + ".class"; + URL url = Thread.currentThread().getContextClassLoader().getResource(file); + String baseDir = url.toURI().getPath(); + baseDir = baseDir.substring(0, baseDir.length() - file.length() - 1); + return baseDir; + } + + /** + * Performs complete setup of SSL configuration in preparation for testing an + * SSLFactory. This includes keys, certs, keystores, truststores, the server + * SSL configuration file, the client SSL configuration file, and the master + * configuration file read by the SSLFactory. + * + * @param keystoresDir String directory to save keystores + * @param sslConfDir String directory to save SSL configuration files + * @param conf Configuration master configuration to be used by an SSLFactory, + * which will be mutated by this method + * @param useClientCert boolean true to make the client present a cert in the + * SSL handshake + */ + public static void setupSSLConfig(String keystoresDir, String sslConfDir, Configuration conf, + boolean useClientCert) throws Exception { + String clientKS = keystoresDir + "/clientKS.jks"; + String clientPassword = "clientP"; + String serverKS = keystoresDir + "/serverKS.jks"; + String serverPassword = "serverP"; + String trustKS = keystoresDir + "/trustKS.jks"; + String trustPassword = "trustP"; + + File sslClientConfFile = new File(sslConfDir + "/ssl-client.xml"); + File sslServerConfFile = new File(sslConfDir + "/ssl-server.xml"); + + Map<String, X509Certificate> certs = new HashMap<>(); + + if (useClientCert) { + KeyPair cKP = generateKeyPair("RSA"); + X509Certificate cCert = + generateCertificate("CN=localhost, O=client", cKP, 30, "SHA1withRSA"); + createKeyStore(clientKS, clientPassword, "client", cKP.getPrivate(), cCert); + certs.put("client", cCert); + } + + KeyPair sKP = generateKeyPair("RSA"); + X509Certificate sCert = + generateCertificate("CN=localhost, O=server", sKP, 30, "SHA1withRSA"); + createKeyStore(serverKS, serverPassword, "server", sKP.getPrivate(), sCert); + certs.put("server", sCert); + + createTrustStore(trustKS, trustPassword, certs); + + Configuration clientSSLConf = + createClientSSLConfig(clientKS, clientPassword, clientPassword, trustKS); + Configuration serverSSLConf = + createServerSSLConfig(serverKS, serverPassword, serverPassword, trustKS); + + saveConfig(sslClientConfFile, clientSSLConf); + saveConfig(sslServerConfFile, serverSSLConf); + + conf.set(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY, "ALLOW_ALL"); + conf.set(SSLFactory.SSL_CLIENT_CONF_KEY, sslClientConfFile.getName()); + conf.set(SSLFactory.SSL_SERVER_CONF_KEY, sslServerConfFile.getName()); + conf.setBoolean(SSLFactory.SSL_REQUIRE_CLIENT_CERT_KEY, useClientCert); + } + + /** + * Creates SSL configuration for a client. + * + * @param clientKS String client keystore file + * @param password String store password, or null to avoid setting store + * password + * @param keyPassword String key password, or null to avoid setting key + * password + * @param trustKS String truststore file + * @return Configuration for client SSL + */ + private static Configuration createClientSSLConfig(String clientKS, String password, + String keyPassword, String trustKS) { + Configuration clientSSLConf = + createSSLConfig(SSLFactory.Mode.CLIENT, clientKS, password, keyPassword, trustKS); + return clientSSLConf; + } + + /** + * Creates SSL configuration for a server. + * + * @param serverKS String server keystore file + * @param password String store password, or null to avoid setting store + * password + * @param keyPassword String key password, or null to avoid setting key + * password + * @param trustKS String truststore file + * @return Configuration for server SSL + */ + private static Configuration createServerSSLConfig(String serverKS, String password, + String keyPassword, String trustKS) throws IOException { + Configuration serverSSLConf = + createSSLConfig(SSLFactory.Mode.SERVER, serverKS, password, keyPassword, trustKS); + return serverSSLConf; + } + + /** + * Creates SSL configuration. + * + * @param mode SSLFactory.Mode mode to configure + * @param keystore String keystore file + * @param password String store password, or null to avoid setting store + * password + * @param keyPassword String key password, or null to avoid setting key + * password + * @param trustKS String truststore file + * @return Configuration for SSL + */ + private static Configuration createSSLConfig(SSLFactory.Mode mode, String keystore, + String password, String keyPassword, String trustKS) { + String trustPassword = "trustP"; + + Configuration sslConf = new Configuration(false); + if (keystore != null) { + sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode, + FileBasedKeyStoresFactory.SSL_KEYSTORE_LOCATION_TPL_KEY), keystore); + } + if (password != null) { + sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode, + FileBasedKeyStoresFactory.SSL_KEYSTORE_PASSWORD_TPL_KEY), password); + } + if (keyPassword != null) { + sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode, + FileBasedKeyStoresFactory.SSL_KEYSTORE_KEYPASSWORD_TPL_KEY), keyPassword); + } + if (trustKS != null) { + sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode, + FileBasedKeyStoresFactory.SSL_TRUSTSTORE_LOCATION_TPL_KEY), trustKS); + } + if (trustPassword != null) { + sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode, + FileBasedKeyStoresFactory.SSL_TRUSTSTORE_PASSWORD_TPL_KEY), trustPassword); + } + sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode, + FileBasedKeyStoresFactory.SSL_TRUSTSTORE_RELOAD_INTERVAL_TPL_KEY), "1000"); + + return sslConf; + } + + /** + * Saves configuration to a file. + * + * @param file File to save + * @param conf Configuration contents to write to file + * @throws IOException if there is an I/O error saving the file + */ + private static void saveConfig(File file, Configuration conf) throws IOException { + Writer writer = new FileWriter(file); + try { + conf.writeXml(writer); + } finally { + writer.close(); + } + } + + private static KeyPair generateKeyPair(String algorithm) throws NoSuchAlgorithmException { + KeyPairGenerator keyGen = KeyPairGenerator.getInstance(algorithm); + keyGen.initialize(1024); + return keyGen.genKeyPair(); + } + + private static KeyStore createEmptyKeyStore() throws GeneralSecurityException, IOException { + KeyStore ks = KeyStore.getInstance("JKS"); + ks.load(null, null); // initialize + return ks; } + private static void saveKeyStore(KeyStore ks, String filename, String password) + throws GeneralSecurityException, IOException { + FileOutputStream out = new FileOutputStream(filename); + try { + ks.store(out, password.toCharArray()); + } finally { + out.close(); + } + } + + private static void createKeyStore(String filename, String password, String alias, + Key privateKey, Certificate cert) throws GeneralSecurityException, IOException { + KeyStore ks = createEmptyKeyStore(); + ks.setKeyEntry(alias, privateKey, password.toCharArray(), new Certificate[] { cert }); + saveKeyStore(ks, filename, password); + } + + private static <T extends Certificate> void createTrustStore(String filename, String password, + Map<String, T> certs) throws GeneralSecurityException, IOException { + KeyStore ks = createEmptyKeyStore(); + for (Map.Entry<String, T> cert : certs.entrySet()) { + ks.setCertificateEntry(cert.getKey(), cert.getValue()); + } + saveKeyStore(ks, filename, password); + } } diff --git a/pom.xml b/pom.xml index f7a0cd9..d8a37a2 100644 --- a/pom.xml +++ b/pom.xml @@ -592,6 +592,12 @@ <version>${hadoop-two.version}</version> <scope>test</scope> </dependency> + <dependency> + <groupId>bouncycastle</groupId> + <artifactId>bcprov-jdk15</artifactId> + <version>140</version> + <scope>test</scope> + </dependency> </dependencies> </dependencyManagement> <profiles>