This is an automated email from the ASF dual-hosted git repository. alsuliman pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit ee12055797925c22fbeceb60244e3ad98c5dd570 Author: Ian Maxon <[email protected]> AuthorDate: Tue Oct 11 14:32:11 2022 -0700 [NO ISSUE] Make UDF http client extensible (cherry pick of 0ed28248c3471079ae226a66c267976e271a1d4a) Change-Id: I783ce8c1d888188f8c6d894a5aa435cbe318ec0e Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17239 Tested-by: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: Ian Maxon <[email protected]> Reviewed-by: Wail Alkowaileet <[email protected]> Change-Id: Ib924f177f8ee50b536953667bbe1ba1ee4dc3701 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17365 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Michael Blow <[email protected]> --- .../asterix/common/library/ILibraryManager.java | 7 +++ .../external/library/ExternalLibraryManager.java | 59 ++++++++++++++-------- 2 files changed, 44 insertions(+), 22 deletions(-) diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java index 93fe92dcaa..acbce6d4d6 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java @@ -24,13 +24,16 @@ import java.io.InputStream; import java.net.URI; import java.security.MessageDigest; import java.util.List; +import java.util.function.Function; import org.apache.asterix.common.metadata.DataverseName; import org.apache.asterix.external.ipc.ExternalFunctionResultRouter; +import org.apache.http.impl.client.CloseableHttpClient; import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.exceptions.HyracksException; import org.apache.hyracks.api.io.FileReference; +import org.apache.hyracks.control.nc.NodeControllerService; import org.apache.hyracks.ipc.impl.IPCSystem; public interface ILibraryManager { @@ -59,9 +62,13 @@ public interface ILibraryManager { IPCSystem getIPCI(); + NodeControllerService getNcs(); + MessageDigest download(FileReference targetFile, String authToken, URI libLocation) throws HyracksException; void unzip(FileReference sourceFile, FileReference outputDir) throws IOException; void writeAndForce(FileReference outputFile, InputStream dataStream, byte[] copyBuf) throws IOException; + + void setUploadClient(Function<ILibraryManager, CloseableHttpClient> f); } diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java index f71150a7c6..050d9f0071 100755 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java @@ -49,6 +49,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; @@ -77,7 +78,6 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; -import org.apache.http.ssl.SSLContexts; import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.exceptions.HyracksException; @@ -92,6 +92,7 @@ import org.apache.hyracks.api.util.IoUtil; import org.apache.hyracks.control.common.work.AbstractWork; import org.apache.hyracks.control.nc.NodeControllerService; import org.apache.hyracks.ipc.impl.IPCSystem; +import org.apache.hyracks.ipc.security.NetworkSecurityManager; import org.apache.hyracks.ipc.sockets.PlainSocketChannelFactory; import org.apache.hyracks.util.file.FileUtil; import org.apache.logging.log4j.LogManager; @@ -102,7 +103,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; -public final class ExternalLibraryManager implements ILibraryManager, ILifeCycleComponent { +public class ExternalLibraryManager implements ILibraryManager, ILifeCycleComponent { public static final String LIBRARY_MANAGER_BASE_DIR_NAME = "library"; @@ -140,6 +141,7 @@ public final class ExternalLibraryManager implements ILibraryManager, ILifeCycle private final ExternalFunctionResultRouter router; private final IIOManager ioManager; private boolean sslEnabled; + private Function<ILibraryManager, CloseableHttpClient> uploadClientSupp; public ExternalLibraryManager(NodeControllerService ncs, IPersistedResourceRegistry reg, FileReference appDir, IIOManager ioManager) { @@ -155,6 +157,7 @@ public final class ExternalLibraryManager implements ILibraryManager, ILifeCycle router = new ExternalFunctionResultRouter(); this.sslEnabled = ncs.getConfiguration().isSslEnabled(); this.ioManager = ioManager; + uploadClientSupp = ExternalLibraryManager::defaultHttpClient; } public void initialize(boolean resetStorageData) throws HyracksDataException { @@ -486,6 +489,11 @@ public final class ExternalLibraryManager implements ILibraryManager, ILifeCycle return pythonIPC; } + @Override + public NodeControllerService getNcs() { + return ncs; + } + private static final class DeleteDirectoryWork extends AbstractWork { private final Path path; @@ -630,30 +638,37 @@ public final class ExternalLibraryManager implements ILibraryManager, ILifeCycle } } - //TODO: this should probably be static so it could be reused somewhere else, or made such that the trust store is not - // reloaded from disk on every client intialization? - private CloseableHttpClient newClient() { + public CloseableHttpClient newClient() { if (sslEnabled) { - try { - final INetworkSecurityManager networkSecurityManager = ncs.getNetworkSecurityManager(); - final INetworkSecurityConfig configuration = networkSecurityManager.getConfiguration(); - KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); - try (FileInputStream trustStoreFile = new FileInputStream(configuration.getTrustStoreFile())) { - String ksPassword = configuration.getKeyStorePassword(); - trustStore.load(trustStoreFile, - ksPassword == null || ksPassword.isEmpty() ? null : ksPassword.toCharArray()); - } - SSLContext sslcontext = SSLContexts.custom().loadTrustMaterial(trustStore, null).build(); - SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslcontext, - new String[] { "TLSv1.2" }, null, SSLConnectionSocketFactory.getDefaultHostnameVerifier()); - return HttpClients.custom().setSSLSocketFactory(sslsf).build(); - - } catch (Exception e) { - throw new IllegalStateException(e); - } + return uploadClientSupp.apply(this); } else { return HttpClients.createDefault(); } } + @Override + public void setUploadClient(Function<ILibraryManager, CloseableHttpClient> f) { + uploadClientSupp = f; + } + + private static CloseableHttpClient defaultHttpClient(ILibraryManager extLib) { + try { + final INetworkSecurityManager networkSecurityManager = extLib.getNcs().getNetworkSecurityManager(); + final INetworkSecurityConfig configuration = networkSecurityManager.getConfiguration(); + KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); + try (FileInputStream trustStoreFile = new FileInputStream(configuration.getTrustStoreFile())) { + String ksPassword = configuration.getKeyStorePassword(); + trustStore.load(trustStoreFile, + ksPassword == null || ksPassword.isEmpty() ? null : ksPassword.toCharArray()); + } + SSLContext sslcontext = NetworkSecurityManager.newSSLContext(configuration); + SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslcontext, new String[] { "TLSv1.2" }, + null, SSLConnectionSocketFactory.getDefaultHostnameVerifier()); + return HttpClients.custom().setSSLSocketFactory(sslsf).build(); + + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + }
