>From Ian Maxon <[email protected]>: Ian Maxon has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17239 )
Change subject: [NO ISSUE] Make UDF http client extensible ...................................................................... [NO ISSUE] Make UDF http client extensible Change-Id: I783ce8c1d888188f8c6d894a5aa435cbe318ec0e Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17239 Tested-by: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: Ian Maxon <[email protected]> Reviewed-by: Wail Alkowaileet <[email protected]> --- M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java 2 files changed, 58 insertions(+), 22 deletions(-) Approvals: Ian Maxon: Looks good to me, but someone else must approve Wail Alkowaileet: Looks good to me, approved Jenkins: Verified; Verified diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java index 93fe92d..acbce6d 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java @@ -24,13 +24,16 @@ import java.net.URI; import java.security.MessageDigest; import java.util.List; +import java.util.function.Function; import org.apache.asterix.common.metadata.DataverseName; import org.apache.asterix.external.ipc.ExternalFunctionResultRouter; +import org.apache.http.impl.client.CloseableHttpClient; import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.exceptions.HyracksException; import org.apache.hyracks.api.io.FileReference; +import org.apache.hyracks.control.nc.NodeControllerService; import org.apache.hyracks.ipc.impl.IPCSystem; public interface ILibraryManager { @@ -59,9 +62,13 @@ IPCSystem getIPCI(); + NodeControllerService getNcs(); + MessageDigest download(FileReference targetFile, String authToken, URI libLocation) throws HyracksException; void unzip(FileReference sourceFile, FileReference outputDir) throws IOException; void writeAndForce(FileReference outputFile, InputStream dataStream, byte[] copyBuf) throws IOException; + + void setUploadClient(Function<ILibraryManager, CloseableHttpClient> f); } diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java index f71150a..050d9f0 100755 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java @@ -49,6 +49,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; @@ -77,7 +78,6 @@ import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; -import org.apache.http.ssl.SSLContexts; import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.exceptions.HyracksException; @@ -92,6 +92,7 @@ import org.apache.hyracks.control.common.work.AbstractWork; import org.apache.hyracks.control.nc.NodeControllerService; import org.apache.hyracks.ipc.impl.IPCSystem; +import org.apache.hyracks.ipc.security.NetworkSecurityManager; import org.apache.hyracks.ipc.sockets.PlainSocketChannelFactory; import org.apache.hyracks.util.file.FileUtil; import org.apache.logging.log4j.LogManager; @@ -102,7 +103,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; -public final class ExternalLibraryManager implements ILibraryManager, ILifeCycleComponent { +public class ExternalLibraryManager implements ILibraryManager, ILifeCycleComponent { public static final String LIBRARY_MANAGER_BASE_DIR_NAME = "library"; @@ -140,6 +141,7 @@ private final ExternalFunctionResultRouter router; private final IIOManager ioManager; private boolean sslEnabled; + private Function<ILibraryManager, CloseableHttpClient> uploadClientSupp; public ExternalLibraryManager(NodeControllerService ncs, IPersistedResourceRegistry reg, FileReference appDir, IIOManager ioManager) { @@ -155,6 +157,7 @@ router = new ExternalFunctionResultRouter(); this.sslEnabled = ncs.getConfiguration().isSslEnabled(); this.ioManager = ioManager; + uploadClientSupp = ExternalLibraryManager::defaultHttpClient; } public void initialize(boolean resetStorageData) throws HyracksDataException { @@ -486,6 +489,11 @@ return pythonIPC; } + @Override + public NodeControllerService getNcs() { + return ncs; + } + private static final class DeleteDirectoryWork extends AbstractWork { private final Path path; @@ -630,30 +638,37 @@ } } - //TODO: this should probably be static so it could be reused somewhere else, or made such that the trust store is not - // reloaded from disk on every client intialization? - private CloseableHttpClient newClient() { + public CloseableHttpClient newClient() { if (sslEnabled) { - try { - final INetworkSecurityManager networkSecurityManager = ncs.getNetworkSecurityManager(); - final INetworkSecurityConfig configuration = networkSecurityManager.getConfiguration(); - KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); - try (FileInputStream trustStoreFile = new FileInputStream(configuration.getTrustStoreFile())) { - String ksPassword = configuration.getKeyStorePassword(); - trustStore.load(trustStoreFile, - ksPassword == null || ksPassword.isEmpty() ? null : ksPassword.toCharArray()); - } - SSLContext sslcontext = SSLContexts.custom().loadTrustMaterial(trustStore, null).build(); - SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslcontext, - new String[] { "TLSv1.2" }, null, SSLConnectionSocketFactory.getDefaultHostnameVerifier()); - return HttpClients.custom().setSSLSocketFactory(sslsf).build(); - - } catch (Exception e) { - throw new IllegalStateException(e); - } + return uploadClientSupp.apply(this); } else { return HttpClients.createDefault(); } } + @Override + public void setUploadClient(Function<ILibraryManager, CloseableHttpClient> f) { + uploadClientSupp = f; + } + + private static CloseableHttpClient defaultHttpClient(ILibraryManager extLib) { + try { + final INetworkSecurityManager networkSecurityManager = extLib.getNcs().getNetworkSecurityManager(); + final INetworkSecurityConfig configuration = networkSecurityManager.getConfiguration(); + KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); + try (FileInputStream trustStoreFile = new FileInputStream(configuration.getTrustStoreFile())) { + String ksPassword = configuration.getKeyStorePassword(); + trustStore.load(trustStoreFile, + ksPassword == null || ksPassword.isEmpty() ? null : ksPassword.toCharArray()); + } + SSLContext sslcontext = NetworkSecurityManager.newSSLContext(configuration); + SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslcontext, new String[] { "TLSv1.2" }, + null, SSLConnectionSocketFactory.getDefaultHostnameVerifier()); + return HttpClients.custom().setSSLSocketFactory(sslsf).build(); + + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + } -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17239 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I783ce8c1d888188f8c6d894a5aa435cbe318ec0e Gerrit-Change-Number: 17239 Gerrit-PatchSet: 3 Gerrit-Owner: Ian Maxon <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Hussain Towaileb <[email protected]> Gerrit-Reviewer: Ian Maxon <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Murtadha Hubail <[email protected]> Gerrit-Reviewer: Wail Alkowaileet <[email protected]> Gerrit-MessageType: merged
