This is an automated email from the ASF dual-hosted git repository. mhubail pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push: new 528c28a8a5 [NO ISSUE][HYR] Enable extension of network security manager new 08a7e4dca2 Merge branch 'gerrit/trinity' into 'master' 528c28a8a5 is described below commit 528c28a8a50aeb32b49401106a5c964f3e785c46 Author: Michael Blow <michael.b...@couchbase.com> AuthorDate: Sun Feb 11 23:01:50 2024 -0500 [NO ISSUE][HYR] Enable extension of network security manager - remove some premature resolution of configured hostnames to ip address Change-Id: Idad460b5894eeed5ef9b43d666d10cfd2e1e4cd6 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18159 Reviewed-by: Murtadha Hubail <mhub...@apache.org> Tested-by: Michael Blow <mb...@apache.org> --- .../org/apache/asterix/test/storage/DeallocatableTest.java | 6 +++--- .../asterix/replication/management/NetworkingUtil.java | 14 -------------- .../java/org/apache/hyracks/api/comm/NetworkAddress.java | 11 +++++------ .../org/apache/hyracks/client/result/ResultSetReader.java | 6 +----- .../hyracks/control/cc/ClusterControllerService.java | 12 ++++++------ .../apache/hyracks/control/nc/NodeControllerService.java | 12 ++++++------ .../control/nc/work/ReportPartitionAvailabilityWork.java | 11 ++--------- .../org/apache/hyracks/control/nc/work/StartTasksWork.java | 11 +++-------- .../hyracks/ipc/security/NetworkSecurityManager.java | 2 +- 9 files changed, 27 insertions(+), 58 deletions(-) diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/DeallocatableTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/DeallocatableTest.java index cfd251b025..368b17bef2 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/DeallocatableTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/DeallocatableTest.java @@ -22,9 +22,9 @@ import java.nio.ByteBuffer; import java.util.concurrent.TimeUnit; import org.apache.asterix.app.bootstrap.TestNodeController; -import org.apache.asterix.replication.management.NetworkingUtil; import org.apache.asterix.test.common.TestHelper; import org.apache.asterix.test.runtime.LangExecutionUtil; +import org.apache.hyracks.api.comm.NetworkAddress; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.dataflow.ConnectorDescriptorId; import org.apache.hyracks.api.dataflow.TaskAttemptId; @@ -63,8 +63,8 @@ public class DeallocatableTest { final IHyracksTaskContext ctx = nc.createTestContext(jobId, 0, true); final ConnectorDescriptorId codId = new ConnectorDescriptorId(1); final PartitionId pid = new PartitionId(ctx.getJobletContext().getJobId(), codId, 1, 1); - final ChannelControlBlock ccb = ncs.getNetworkManager() - .connect(NetworkingUtil.getSocketAddress(ncs.getNetworkManager().getLocalNetworkAddress())); + NetworkAddress netAddr = ncs.getNetworkManager().getLocalNetworkAddress(); + final ChannelControlBlock ccb = ncs.getNetworkManager().connect(netAddr.toResolvedInetSocketAddress()); final NetworkOutputChannel networkOutputChannel = new NetworkOutputChannel(ccb, 0); final MaterializingPipelinedPartition mpp = new MaterializingPipelinedPartition(ctx, ncs.getPartitionManager(), pid, taId, ncs.getExecutor()); diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java index b38f0aa8d3..9caaa79ce9 100644 --- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java +++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java @@ -21,18 +21,14 @@ package org.apache.asterix.replication.management; import java.io.EOFException; import java.io.IOException; import java.net.InetAddress; -import java.net.InetSocketAddress; import java.net.NetworkInterface; -import java.net.SocketAddress; import java.net.SocketException; -import java.net.UnknownHostException; import java.nio.ByteBuffer; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; import java.nio.channels.SocketChannel; import java.util.Enumeration; -import org.apache.hyracks.api.comm.NetworkAddress; import org.apache.hyracks.api.network.ISocketChannel; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -131,16 +127,6 @@ public class NetworkingUtil { fileChannel.transferFrom(socketChannel, pos, fileSize); } - public static InetSocketAddress getSocketAddress(SocketChannel socketChannel) { - String hostAddress = socketChannel.socket().getInetAddress().getHostAddress(); - int port = socketChannel.socket().getPort(); - return InetSocketAddress.createUnresolved(hostAddress, port); - } - - public static SocketAddress getSocketAddress(NetworkAddress netAddr) throws UnknownHostException { - return new InetSocketAddress(InetAddress.getByAddress(netAddr.lookupIpAddress()), netAddr.getPort()); - } - public static boolean isHealthy(ISocketChannel sc) { return sc != null && sc.getSocketChannel().isOpen() && sc.getSocketChannel().isConnected(); } diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/comm/NetworkAddress.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/comm/NetworkAddress.java index 75fbb923f3..9f9186b404 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/comm/NetworkAddress.java +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/comm/NetworkAddress.java @@ -35,8 +35,6 @@ public final class NetworkAddress implements IWritable, Serializable { private String address; // Cached locally, not serialized private volatile byte[] ipAddress; - // Cached locally, not serialized - private volatile InetSocketAddress inetSocketAddress; private int port; @@ -76,11 +74,12 @@ public final class NetworkAddress implements IWritable, Serializable { return ipAddress; } - public InetSocketAddress resolveInetSocketAddress() { - if (inetSocketAddress == null) { - inetSocketAddress = new InetSocketAddress(address, port); + public InetSocketAddress toResolvedInetSocketAddress() throws UnknownHostException { + InetSocketAddress addr = toInetSocketAddress(); + if (addr.isUnresolved()) { + throw new UnknownHostException(getAddress()); } - return inetSocketAddress; + return addr; } public InetSocketAddress toInetSocketAddress() { diff --git a/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultSetReader.java b/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultSetReader.java index b29e2ea2d8..992c8bebf3 100644 --- a/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultSetReader.java +++ b/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultSetReader.java @@ -18,8 +18,6 @@ */ package org.apache.hyracks.client.result; -import java.net.InetAddress; -import java.net.InetSocketAddress; import java.net.SocketAddress; import java.net.UnknownHostException; import java.nio.ByteBuffer; @@ -28,7 +26,6 @@ import org.apache.hyracks.api.channels.IInputChannel; import org.apache.hyracks.api.channels.IInputChannelMonitor; import org.apache.hyracks.api.comm.FrameHelper; import org.apache.hyracks.api.comm.IFrame; -import org.apache.hyracks.api.comm.NetworkAddress; import org.apache.hyracks.api.context.IHyracksCommonContext; import org.apache.hyracks.api.exceptions.ErrorCode; import org.apache.hyracks.api.exceptions.HyracksDataException; @@ -146,8 +143,7 @@ public class ResultSetReader implements IResultSetReader { private SocketAddress getSocketAddress(ResultDirectoryRecord record) throws HyracksDataException { try { - final NetworkAddress netAddr = record.getNetworkAddress(); - return new InetSocketAddress(InetAddress.getByAddress(netAddr.lookupIpAddress()), netAddr.getPort()); + return record.getNetworkAddress().toResolvedInetSocketAddress(); } catch (UnknownHostException e) { throw HyracksDataException.create(e); } diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java index f11e7ffb99..d6698fe703 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java @@ -35,6 +35,7 @@ import java.util.TimerTask; import java.util.TreeMap; import java.util.concurrent.ExecutorService; +import org.apache.hyracks.api.application.IApplication; import org.apache.hyracks.api.application.ICCApplication; import org.apache.hyracks.api.client.ClusterControllerInfo; import org.apache.hyracks.api.comm.NetworkAddress; @@ -49,7 +50,6 @@ import org.apache.hyracks.api.job.JobId; import org.apache.hyracks.api.job.JobIdFactory; import org.apache.hyracks.api.job.JobParameterByteStore; import org.apache.hyracks.api.job.resource.IJobCapacityController; -import org.apache.hyracks.api.network.INetworkSecurityConfig; import org.apache.hyracks.api.network.INetworkSecurityManager; import org.apache.hyracks.api.service.IControllerService; import org.apache.hyracks.api.topology.ClusterTopology; @@ -170,8 +170,7 @@ public class ClusterControllerService implements IControllerService { File jobLogFolder = new File(ccConfig.getRootDir(), "logs/jobs"); jobLog = new LogFile(jobLogFolder); - final INetworkSecurityConfig securityConfig = getNetworkSecurityConfig(); - networkSecurityManager = new NetworkSecurityManager(securityConfig); + networkSecurityManager = createNetworkSecurityManager(ccConfig.getAppConfig(), application); // WorkQueue is in charge of heartbeat as well as other events. workQueue = new WorkQueue("ClusterController", Thread.MAX_PRIORITY); @@ -567,8 +566,9 @@ public class ClusterControllerService implements IControllerService { return networkSecurityManager; } - protected INetworkSecurityConfig getNetworkSecurityConfig() { - return NetworkSecurityConfig.of(ccConfig.isSslEnabled(), ccConfig.getKeyStorePath(), - ccConfig.getKeyStorePassword(), ccConfig.getTrustStorePath()); + protected INetworkSecurityManager createNetworkSecurityManager(IApplicationConfig appConfig, IApplication app) + throws Exception { + return new NetworkSecurityManager(NetworkSecurityConfig.of(ccConfig.isSslEnabled(), ccConfig.getKeyStorePath(), + ccConfig.getKeyStorePassword(), ccConfig.getTrustStorePath())); } } diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java index f69d106f1d..e173dcb4a6 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java @@ -45,6 +45,7 @@ import org.apache.hyracks.api.application.INCApplication; import org.apache.hyracks.api.client.NodeControllerInfo; import org.apache.hyracks.api.client.NodeStatus; import org.apache.hyracks.api.comm.NetworkAddress; +import org.apache.hyracks.api.config.IApplicationConfig; import org.apache.hyracks.api.control.CcId; import org.apache.hyracks.api.deployment.DeploymentId; import org.apache.hyracks.api.exceptions.ErrorCode; @@ -56,7 +57,6 @@ import org.apache.hyracks.api.job.JobId; import org.apache.hyracks.api.job.JobParameterByteStore; import org.apache.hyracks.api.lifecycle.ILifeCycleComponentManager; import org.apache.hyracks.api.lifecycle.LifeCycleComponentManager; -import org.apache.hyracks.api.network.INetworkSecurityConfig; import org.apache.hyracks.api.network.INetworkSecurityManager; import org.apache.hyracks.api.result.IResultPartitionManager; import org.apache.hyracks.api.service.IControllerService; @@ -197,8 +197,7 @@ public class NodeControllerService implements IControllerService { if (application == null) { throw new IllegalArgumentException("INCApplication cannot be null"); } - final INetworkSecurityConfig securityConfig = getNetworkSecurityConfig(); - networkSecurityManager = new NetworkSecurityManager(securityConfig); + networkSecurityManager = createNetworkSecurityManager(ncConfig.getAppConfig(), application); this.application = application; id = ncConfig.getNodeId(); if (id == null) { @@ -726,8 +725,9 @@ public class NodeControllerService implements IControllerService { return networkSecurityManager; } - protected INetworkSecurityConfig getNetworkSecurityConfig() { - return NetworkSecurityConfig.of(ncConfig.isSslEnabled(), ncConfig.getKeyStorePath(), - ncConfig.getKeyStorePassword(), ncConfig.getTrustStorePath()); + protected INetworkSecurityManager createNetworkSecurityManager(IApplicationConfig appConfig, + INCApplication application) { + return new NetworkSecurityManager(NetworkSecurityConfig.of(ncConfig.isSslEnabled(), ncConfig.getKeyStorePath(), + ncConfig.getKeyStorePassword(), ncConfig.getTrustStorePath())); } } diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/ReportPartitionAvailabilityWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/ReportPartitionAvailabilityWork.java index cfd69ce5d1..a6952b95c0 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/ReportPartitionAvailabilityWork.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/ReportPartitionAvailabilityWork.java @@ -18,8 +18,6 @@ */ package org.apache.hyracks.control.nc.work; -import java.net.InetAddress; -import java.net.InetSocketAddress; import java.util.Map; import org.apache.hyracks.api.comm.NetworkAddress; @@ -50,13 +48,8 @@ public class ReportPartitionAvailabilityWork extends AbstractWork { Map<JobId, Joblet> jobletMap = ncs.getJobletMap(); Joblet ji = jobletMap.get(pid.getJobId()); if (ji != null) { - PartitionChannel channel = - new PartitionChannel(pid, - new NetworkInputChannel(ncs.getNetworkManager(), - new InetSocketAddress( - InetAddress.getByAddress(networkAddress.lookupIpAddress()), - networkAddress.getPort()), - pid, 5)); + PartitionChannel channel = new PartitionChannel(pid, new NetworkInputChannel(ncs.getNetworkManager(), + networkAddress.toResolvedInetSocketAddress(), pid, 5)); ji.reportPartitionAvailability(channel); } } catch (Exception e) { diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/StartTasksWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/StartTasksWork.java index dd4a956bd3..6225d4cc49 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/StartTasksWork.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/StartTasksWork.java @@ -18,8 +18,6 @@ */ package org.apache.hyracks.control.nc.work; -import java.net.InetAddress; -import java.net.InetSocketAddress; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; @@ -301,12 +299,9 @@ public class StartTasksWork extends AbstractWork { NetworkAddress networkAddress = inputAddresses[i][j]; PartitionId pid = new PartitionId(jobId, inputs.get(i).getConnectorId(), j, td.getTaskAttemptId().getTaskId().getPartition()); - PartitionChannel channel = new PartitionChannel(pid, - new NetworkInputChannel(ncs.getNetworkManager(), - new InetSocketAddress( - InetAddress.getByAddress(networkAddress.lookupIpAddress()), - networkAddress.getPort()), - pid, 5)); + PartitionChannel channel = + new PartitionChannel(pid, new NetworkInputChannel(ncs.getNetworkManager(), + networkAddress.toResolvedInetSocketAddress(), pid, 5)); channels.add(channel); } } diff --git a/hyracks-fullstack/hyracks/hyracks-ipc/src/main/java/org/apache/hyracks/ipc/security/NetworkSecurityManager.java b/hyracks-fullstack/hyracks/hyracks-ipc/src/main/java/org/apache/hyracks/ipc/security/NetworkSecurityManager.java index 42dacf573c..db524ca3d5 100644 --- a/hyracks-fullstack/hyracks/hyracks-ipc/src/main/java/org/apache/hyracks/ipc/security/NetworkSecurityManager.java +++ b/hyracks-fullstack/hyracks/hyracks-ipc/src/main/java/org/apache/hyracks/ipc/security/NetworkSecurityManager.java @@ -36,7 +36,7 @@ import org.apache.hyracks.ipc.sockets.SslSocketChannelFactory; public class NetworkSecurityManager implements INetworkSecurityManager { private volatile INetworkSecurityConfig config; - private final ISocketChannelFactory sslSocketFactory; + protected final ISocketChannelFactory sslSocketFactory; public static final String TSL_VERSION = "TLSv1.2"; public NetworkSecurityManager(INetworkSecurityConfig config) {