Repository: reef Updated Branches: refs/heads/master cf1d65150 -> 445b85d75
[REEF-347] Configure .NET tests to only listen on 127.0.0.1 This addressed the issue by * Adding an environment-dependent DriverConfigurationProvider. * Have C# Local Runtime processes listen only on the loopback address. JIRA: [REEF-347](https://issues.apache.org/jira/browse/REEF-347) Pull Request: This closes #896 Project: http://git-wip-us.apache.org/repos/asf/reef/repo Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/445b85d7 Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/445b85d7 Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/445b85d7 Branch: refs/heads/master Commit: 445b85d75e1ec5bdcaa2d9e1f06a90aa6488de5f Parents: cf1d651 Author: Andrew Chung <[email protected]> Authored: Mon Mar 21 16:24:46 2016 -0700 Committer: Markus Weimer <[email protected]> Committed: Fri Apr 8 11:37:12 2016 -0700 ---------------------------------------------------------------------- .../Common/DriverFolderPreparationHelper.cs | 17 ++++-- .../EnvironmentDriverConfigurationProviders.cs | 34 ++++++++++++ .../Org.Apache.REEF.Client/Local/LocalClient.cs | 6 +++ .../Local/LocalDriverConfigurationProvider.cs | 48 +++++++++++++++++ .../Local/LocalRuntimeClientConfiguration.cs | 4 ++ .../Org.Apache.REEF.Client.csproj | 2 + .../NetworkService/NetworkServiceTests.cs | 2 +- .../Naming/NameServer.cs | 4 +- .../NetworkService/NetworkService.cs | 7 +-- .../NetworkService/StreamingNetworkService.cs | 5 +- .../Org.Apache.REEF.Wake.csproj | 3 ++ .../Remote/ILocalAddressProvider.cs | 32 ++++++++++++ .../Remote/Impl/DefaultLocalAddressProvider.cs | 37 ++++++++++++++ .../Remote/Impl/DefaultRemoteManager.cs | 8 ++- .../Remote/Impl/DefaultRemoteManagerFactory.cs | 9 +++- .../cs/Org.Apache.REEF.Wake/Remote/Impl/Link.cs | 15 +----- .../Remote/Impl/LoopbackLocalAddressProvider.cs | 54 ++++++++++++++++++++ .../bridge/client/LocalSubmissionFromCS.java | 2 + .../address/LoopbackLocalAddressProvider.java | 51 ++++++++++++++++++ .../apache/reef/webserver/HttpServerImpl.java | 19 +++++-- 20 files changed, 328 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/reef/blob/445b85d7/lang/cs/Org.Apache.REEF.Client/Common/DriverFolderPreparationHelper.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Common/DriverFolderPreparationHelper.cs b/lang/cs/Org.Apache.REEF.Client/Common/DriverFolderPreparationHelper.cs index 0e31d33..73fe033 100644 --- a/lang/cs/Org.Apache.REEF.Client/Common/DriverFolderPreparationHelper.cs +++ b/lang/cs/Org.Apache.REEF.Client/Common/DriverFolderPreparationHelper.cs @@ -15,17 +15,22 @@ // specific language governing permissions and limitations // under the License. -using System; +using System.Collections.Generic; using System.IO; using System.Linq; using Org.Apache.REEF.Client.API; -using Org.Apache.REEF.Common; +using Org.Apache.REEF.Common.Client.Parameters; +using Org.Apache.REEF.Common.Evaluator.Parameters; using Org.Apache.REEF.Common.Files; using Org.Apache.REEF.Common.Jar; using Org.Apache.REEF.Tang.Annotations; using Org.Apache.REEF.Tang.Formats; using Org.Apache.REEF.Tang.Implementations.Configuration; +using Org.Apache.REEF.Tang.Implementations.Tang; +using Org.Apache.REEF.Tang.Interface; using Org.Apache.REEF.Utilities.Logging; +using Org.Apache.REEF.Wake.Remote; +using Org.Apache.REEF.Wake.Remote.Impl; namespace Org.Apache.REEF.Client.Common { @@ -54,16 +59,19 @@ namespace Org.Apache.REEF.Client.Common private readonly AvroConfigurationSerializer _configurationSerializer; private readonly REEFFileNames _fileNames; private readonly FileSets _fileSets; + private readonly ISet<IConfigurationProvider> _driverConfigurationProviders; [Inject] internal DriverFolderPreparationHelper( REEFFileNames fileNames, AvroConfigurationSerializer configurationSerializer, - FileSets fileSets) + FileSets fileSets, + [Parameter(typeof(EnvironmentDriverConfigurationProviders))] ISet<IConfigurationProvider> driverConfigurationProviders) { _fileNames = fileNames; _configurationSerializer = configurationSerializer; _fileSets = fileSets; + _driverConfigurationProviders = driverConfigurationProviders; } /// <summary> @@ -102,7 +110,8 @@ namespace Org.Apache.REEF.Client.Common /// <param name="driverFolderPath"></param> internal void CreateDriverConfiguration(AppParameters appParameters, string driverFolderPath) { - var driverConfiguration = Configurations.Merge(appParameters.DriverConfigurations.ToArray()); + var driverConfigurations = _driverConfigurationProviders.Select(configurationProvider => configurationProvider.GetConfiguration()).ToList(); + var driverConfiguration = Configurations.Merge(driverConfigurations.Concat(appParameters.DriverConfigurations).ToArray()); _configurationSerializer.ToFile(driverConfiguration, Path.Combine(driverFolderPath, _fileNames.GetClrDriverConfigurationPath())); http://git-wip-us.apache.org/repos/asf/reef/blob/445b85d7/lang/cs/Org.Apache.REEF.Client/Common/EnvironmentDriverConfigurationProviders.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Common/EnvironmentDriverConfigurationProviders.cs b/lang/cs/Org.Apache.REEF.Client/Common/EnvironmentDriverConfigurationProviders.cs new file mode 100644 index 0000000..ef70682 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/Common/EnvironmentDriverConfigurationProviders.cs @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Collections.Generic; +using Org.Apache.REEF.Tang.Annotations; +using Org.Apache.REEF.Tang.Interface; + +namespace Org.Apache.REEF.Client.Common +{ + /// <summary> + /// This named parameter is used to target receivers Configuration providers at driver level, provided by the Environment. + /// </summary> + [NamedParameter] + internal sealed class EnvironmentDriverConfigurationProviders : Name<ISet<IConfigurationProvider>> + { + private EnvironmentDriverConfigurationProviders() + { + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/445b85d7/lang/cs/Org.Apache.REEF.Client/Local/LocalClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Local/LocalClient.cs b/lang/cs/Org.Apache.REEF.Client/Local/LocalClient.cs index 23e15f7..460e87d 100644 --- a/lang/cs/Org.Apache.REEF.Client/Local/LocalClient.cs +++ b/lang/cs/Org.Apache.REEF.Client/Local/LocalClient.cs @@ -30,9 +30,13 @@ using Org.Apache.REEF.Common.Avro; using Org.Apache.REEF.Common.Files; using Org.Apache.REEF.Tang.Annotations; using Org.Apache.REEF.Tang.Implementations.Tang; +using Org.Apache.REEF.Tang.Interface; +using Org.Apache.REEF.Tang.Util; using Org.Apache.REEF.Utilities.AsyncUtils; using Org.Apache.REEF.Utilities.Attributes; using Org.Apache.REEF.Utilities.Logging; +using Org.Apache.REEF.Wake.Remote; +using Org.Apache.REEF.Wake.Remote.Impl; using Org.Apache.REEF.Wake.Remote.Parameters; namespace Org.Apache.REEF.Client.Local @@ -58,6 +62,7 @@ namespace Org.Apache.REEF.Client.Local private readonly int _maxNumberOfConcurrentEvaluators; private readonly string _runtimeFolder; private readonly REEFFileNames _fileNames; + private readonly IConfiguration _localConfigurationOnDriver; [Inject] private LocalClient(DriverFolderPreparationHelper driverFolderPreparationHelper, @@ -71,6 +76,7 @@ namespace Org.Apache.REEF.Client.Local _maxNumberOfConcurrentEvaluators = maxNumberOfConcurrentEvaluators; _javaClientLauncher = javaClientLauncher; _fileNames = fileNames; + _localConfigurationOnDriver = TangFactory.GetTang().NewConfigurationBuilder().BindImplementation(GenericType<ILocalAddressProvider>.Class, GenericType<LoopbackLocalAddressProvider>.Class).Build(); } /// <summary> http://git-wip-us.apache.org/repos/asf/reef/blob/445b85d7/lang/cs/Org.Apache.REEF.Client/Local/LocalDriverConfigurationProvider.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Local/LocalDriverConfigurationProvider.cs b/lang/cs/Org.Apache.REEF.Client/Local/LocalDriverConfigurationProvider.cs new file mode 100644 index 0000000..9ae5793 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/Local/LocalDriverConfigurationProvider.cs @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Org.Apache.REEF.Common.Evaluator.Parameters; +using Org.Apache.REEF.Tang.Annotations; +using Org.Apache.REEF.Tang.Implementations.Tang; +using Org.Apache.REEF.Tang.Interface; +using Org.Apache.REEF.Wake.Remote; +using Org.Apache.REEF.Wake.Remote.Impl; + +namespace Org.Apache.REEF.Client.Local +{ + /// <summary> + /// The environment driver configuration provider for the Local Runtime. + /// </summary> + internal sealed class LocalDriverConfigurationProvider : IConfigurationProvider + { + private readonly IConfiguration _configuration; + + [Inject] + private LocalDriverConfigurationProvider() + { + _configuration = TangFactory.GetTang().NewConfigurationBuilder() + .BindImplementation<ILocalAddressProvider, LoopbackLocalAddressProvider>() + .BindSetEntry<EvaluatorConfigurationProviders, LoopbackLocalAddressProvider, IConfigurationProvider>() + .Build(); + } + + public IConfiguration GetConfiguration() + { + return _configuration; + } + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/445b85d7/lang/cs/Org.Apache.REEF.Client/Local/LocalRuntimeClientConfiguration.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Local/LocalRuntimeClientConfiguration.cs b/lang/cs/Org.Apache.REEF.Client/Local/LocalRuntimeClientConfiguration.cs index 30e0ffd..c67a133 100644 --- a/lang/cs/Org.Apache.REEF.Client/Local/LocalRuntimeClientConfiguration.cs +++ b/lang/cs/Org.Apache.REEF.Client/Local/LocalRuntimeClientConfiguration.cs @@ -16,8 +16,10 @@ // under the License. using Org.Apache.REEF.Client.API; +using Org.Apache.REEF.Client.Common; using Org.Apache.REEF.Client.Local.Parameters; using Org.Apache.REEF.Tang.Formats; +using Org.Apache.REEF.Tang.Interface; using Org.Apache.REEF.Tang.Util; namespace Org.Apache.REEF.Client.Local @@ -45,6 +47,8 @@ namespace Org.Apache.REEF.Client.Local public static ConfigurationModule ConfigurationModule = new LocalRuntimeClientConfiguration() .BindImplementation(GenericType<IREEFClient>.Class, GenericType<LocalClient>.Class) + .BindSetEntry<EnvironmentDriverConfigurationProviders, LocalDriverConfigurationProvider, IConfigurationProvider>( + GenericType<EnvironmentDriverConfigurationProviders>.Class, GenericType<LocalDriverConfigurationProvider>.Class) .BindNamedParameter(GenericType<LocalRuntimeDirectory>.Class, RuntimeFolder) .BindNamedParameter(GenericType<NumberOfEvaluators>.Class, NumberOfEvaluators) .Build(); http://git-wip-us.apache.org/repos/asf/reef/blob/445b85d7/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj index 67e663b..e2f410b 100644 --- a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj +++ b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj @@ -85,6 +85,7 @@ under the License. <Compile Include="Common\DotNetFile.cs" /> <Compile Include="Common\ClientConstants.cs" /> <Compile Include="Common\DriverFolderPreparationHelper.cs" /> + <Compile Include="Common\EnvironmentDriverConfigurationProviders.cs" /> <Compile Include="Common\IFile.cs" /> <Compile Include="Common\IResourceArchiveFileGenerator.cs" /> <Compile Include="Common\FileSets.cs" /> @@ -94,6 +95,7 @@ under the License. <Compile Include="Common\JavaClientLauncher.cs" /> <Compile Include="Common\ResourceArchiveFileGenerator.cs" /> <Compile Include="Local\LocalClient.cs" /> + <Compile Include="Local\LocalDriverConfigurationProvider.cs" /> <Compile Include="Local\LocalJobSubmissionResult.cs" /> <Compile Include="Local\LocalRuntimeClientConfiguration.cs" /> <Compile Include="Local\Parameters\LocalRuntimeDirectory.cs" /> http://git-wip-us.apache.org/repos/asf/reef/blob/445b85d7/lang/cs/Org.Apache.REEF.Network.Tests/NetworkService/NetworkServiceTests.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Network.Tests/NetworkService/NetworkServiceTests.cs b/lang/cs/Org.Apache.REEF.Network.Tests/NetworkService/NetworkServiceTests.cs index b458710..d62e9af 100644 --- a/lang/cs/Org.Apache.REEF.Network.Tests/NetworkService/NetworkServiceTests.cs +++ b/lang/cs/Org.Apache.REEF.Network.Tests/NetworkService/NetworkServiceTests.cs @@ -158,7 +158,7 @@ namespace Org.Apache.REEF.Network.Tests.NetworkService var nameClient = injector.GetInstance<NameClient>(); var remoteManagerFactory = injector.GetInstance<IRemoteManagerFactory>(); return new NetworkService<string>(networkServicePort, - handler, new StringIdentifierFactory(), new StringCodec(), nameClient, remoteManagerFactory); + handler, new StringIdentifierFactory(), new StringCodec(), nameClient, injector.GetInstance<ILocalAddressProvider>(), remoteManagerFactory); } private class MessageHandler : IObserver<NsMessage<string>> http://git-wip-us.apache.org/repos/asf/reef/blob/445b85d7/lang/cs/Org.Apache.REEF.Network/Naming/NameServer.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Network/Naming/NameServer.cs b/lang/cs/Org.Apache.REEF.Network/Naming/NameServer.cs index ff34602..d122c0f 100644 --- a/lang/cs/Org.Apache.REEF.Network/Naming/NameServer.cs +++ b/lang/cs/Org.Apache.REEF.Network/Naming/NameServer.cs @@ -49,12 +49,14 @@ namespace Org.Apache.REEF.Network.Naming /// Create a new NameServer to run on the specified port. /// </summary> /// <param name="port">The port to listen for incoming connections on.</param> + /// <param name="addressProvider">The address provider.</param> /// <param name="tcpPortProvider">If port is 0, this interface provides /// a port range to try. /// </param> [Inject] private NameServer( [Parameter(typeof(NamingConfigurationOptions.NameServerPort))] int port, + ILocalAddressProvider addressProvider, ITcpPortProvider tcpPortProvider) { IObserver<TransportEvent<NamingEvent>> handler = CreateServerHandler(); @@ -64,7 +66,7 @@ namespace Org.Apache.REEF.Network.Naming // Start transport server, get listening IP endpoint _logger.Log(Level.Info, "Starting naming server"); _server = new TransportServer<NamingEvent>( - new IPEndPoint(NetworkUtils.LocalIPAddress, port), handler, + new IPEndPoint(addressProvider.LocalAddress, port), handler, codec, tcpPortProvider); _server.Run(); LocalEndpoint = _server.LocalEndpoint; http://git-wip-us.apache.org/repos/asf/reef/blob/445b85d7/lang/cs/Org.Apache.REEF.Network/NetworkService/NetworkService.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Network/NetworkService/NetworkService.cs b/lang/cs/Org.Apache.REEF.Network/NetworkService/NetworkService.cs index f63782d..90cec27 100644 --- a/lang/cs/Org.Apache.REEF.Network/NetworkService/NetworkService.cs +++ b/lang/cs/Org.Apache.REEF.Network/NetworkService/NetworkService.cs @@ -25,7 +25,6 @@ using Org.Apache.REEF.Tang.Exceptions; using Org.Apache.REEF.Utilities.Logging; using Org.Apache.REEF.Wake; using Org.Apache.REEF.Wake.Remote; -using Org.Apache.REEF.Wake.Util; namespace Org.Apache.REEF.Network.NetworkService { @@ -42,7 +41,7 @@ namespace Org.Apache.REEF.Network.NetworkService private readonly ICodec<NsMessage<T>> _codec; private IIdentifier _localIdentifier; private IDisposable _messageHandlerDisposable; - private readonly Dictionary<IIdentifier, IConnection<T>> _connectionMap; + private readonly Dictionary<IIdentifier, IConnection<T>> _connectionMap; /// <summary> /// Create a new NetworkService. @@ -52,6 +51,7 @@ namespace Org.Apache.REEF.Network.NetworkService /// <param name="idFactory">The factory used to create IIdentifiers</param> /// <param name="codec">The codec used for serialization</param> /// <param name="nameClient"></param> + /// <param name="localAddressProvider">The local address provider</param> /// <param name="remoteManagerFactory">Used to instantiate remote manager instances.</param> [Inject] public NetworkService( @@ -60,11 +60,12 @@ namespace Org.Apache.REEF.Network.NetworkService IIdentifierFactory idFactory, ICodec<T> codec, INameClient nameClient, + ILocalAddressProvider localAddressProvider, IRemoteManagerFactory remoteManagerFactory) { _codec = new NsMessageCodec<T>(codec, idFactory); - IPAddress localAddress = NetworkUtils.LocalIPAddress; + IPAddress localAddress = localAddressProvider.LocalAddress; _remoteManager = remoteManagerFactory.GetInstance(localAddress, nsPort, _codec); _messageHandler = messageHandler; http://git-wip-us.apache.org/repos/asf/reef/blob/445b85d7/lang/cs/Org.Apache.REEF.Network/NetworkService/StreamingNetworkService.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Network/NetworkService/StreamingNetworkService.cs b/lang/cs/Org.Apache.REEF.Network/NetworkService/StreamingNetworkService.cs index 1dac39a..94c7c1c 100644 --- a/lang/cs/Org.Apache.REEF.Network/NetworkService/StreamingNetworkService.cs +++ b/lang/cs/Org.Apache.REEF.Network/NetworkService/StreamingNetworkService.cs @@ -55,6 +55,7 @@ namespace Org.Apache.REEF.Network.NetworkService /// <param name="remoteManagerFactory">Writable RemoteManagerFactory to create a /// Writable RemoteManager</param> /// <param name="codec">Codec for Network Service message</param> + /// <param name="localAddressProvider">The local address provider</param> /// <param name="injector">Fork of the injector that created the Network service</param> [Inject] private StreamingNetworkService( @@ -63,10 +64,10 @@ namespace Org.Apache.REEF.Network.NetworkService INameClient nameClient, StreamingRemoteManagerFactory remoteManagerFactory, NsMessageStreamingCodec<T> codec, + ILocalAddressProvider localAddressProvider, IInjector injector) { - IPAddress localAddress = NetworkUtils.LocalIPAddress; - _remoteManager = remoteManagerFactory.GetInstance(localAddress, codec); + _remoteManager = remoteManagerFactory.GetInstance(localAddressProvider.LocalAddress, codec); // Create and register incoming message handler // TODO[REEF-419] This should use the TcpPortProvider mechanism http://git-wip-us.apache.org/repos/asf/reef/blob/445b85d7/lang/cs/Org.Apache.REEF.Wake/Org.Apache.REEF.Wake.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Wake/Org.Apache.REEF.Wake.csproj b/lang/cs/Org.Apache.REEF.Wake/Org.Apache.REEF.Wake.csproj index 1d6551e..f25c6bf 100644 --- a/lang/cs/Org.Apache.REEF.Wake/Org.Apache.REEF.Wake.csproj +++ b/lang/cs/Org.Apache.REEF.Wake/Org.Apache.REEF.Wake.csproj @@ -53,6 +53,9 @@ under the License. <Compile Include="IIdentifierFactory.cs" /> <Compile Include="Remote\IConnectionRetryHandler.cs" /> <Compile Include="Remote\Impl\RemoteConnectionRetryHandler.cs" /> + <Compile Include="Remote\ILocalAddressProvider.cs" /> + <Compile Include="Remote\Impl\DefaultLocalAddressProvider.cs" /> + <Compile Include="Remote\Impl\LoopbackLocalAddressProvider.cs" /> <Compile Include="Remote\Impl\RemoteEventStreamingCodec.cs" /> <Compile Include="Remote\Impl\StreamingRemoteManagerFactory.cs" /> <Compile Include="Remote\Impl\DefaultRemoteManagerFactory.cs" /> http://git-wip-us.apache.org/repos/asf/reef/blob/445b85d7/lang/cs/Org.Apache.REEF.Wake/Remote/ILocalAddressProvider.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Wake/Remote/ILocalAddressProvider.cs b/lang/cs/Org.Apache.REEF.Wake/Remote/ILocalAddressProvider.cs new file mode 100644 index 0000000..5019591 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Wake/Remote/ILocalAddressProvider.cs @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Net; +using Org.Apache.REEF.Tang.Annotations; +using Org.Apache.REEF.Wake.Remote.Impl; + +namespace Org.Apache.REEF.Wake.Remote +{ + /// <summary> + /// Provides the local address. + /// </summary> + [DefaultImplementation(typeof(DefaultLocalAddressProvider))] + public interface ILocalAddressProvider + { + IPAddress LocalAddress { get; } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/445b85d7/lang/cs/Org.Apache.REEF.Wake/Remote/Impl/DefaultLocalAddressProvider.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Wake/Remote/Impl/DefaultLocalAddressProvider.cs b/lang/cs/Org.Apache.REEF.Wake/Remote/Impl/DefaultLocalAddressProvider.cs new file mode 100644 index 0000000..d238e37 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Wake/Remote/Impl/DefaultLocalAddressProvider.cs @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Net; +using Org.Apache.REEF.Tang.Annotations; +using Org.Apache.REEF.Wake.Util; + +namespace Org.Apache.REEF.Wake.Remote.Impl +{ + /// <summary> + /// The local address provider that provides the local IP address using <see cref="NetworkUtils.LocalIPAddress"/>. + /// </summary> + public sealed class DefaultLocalAddressProvider : ILocalAddressProvider + { + [Inject] + private DefaultLocalAddressProvider() + { + LocalAddress = NetworkUtils.LocalIPAddress; + } + + public IPAddress LocalAddress { get; private set; } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/445b85d7/lang/cs/Org.Apache.REEF.Wake/Remote/Impl/DefaultRemoteManager.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Wake/Remote/Impl/DefaultRemoteManager.cs b/lang/cs/Org.Apache.REEF.Wake/Remote/Impl/DefaultRemoteManager.cs index 6a96d48..32f388b 100644 --- a/lang/cs/Org.Apache.REEF.Wake/Remote/Impl/DefaultRemoteManager.cs +++ b/lang/cs/Org.Apache.REEF.Wake/Remote/Impl/DefaultRemoteManager.cs @@ -85,9 +85,13 @@ namespace Org.Apache.REEF.Wake.Remote.Impl /// <summary> /// Constructs a DefaultRemoteManager. Does not listen for incoming messages. /// </summary> + /// <param name="localAddressProvider">The local address provider</param> /// <param name="codec">The codec used for serializing messages</param> /// <param name="tcpClientFactory">provides TcpClient for given endpoint</param> - internal DefaultRemoteManager(ICodec<T> codec, ITcpClientConnectionFactory tcpClientFactory) + internal DefaultRemoteManager( + ILocalAddressProvider localAddressProvider, + ICodec<T> codec, + ITcpClientConnectionFactory tcpClientFactory) { using (LOGGER.LogFunction("DefaultRemoteManager::DefaultRemoteManager")) { @@ -101,7 +105,7 @@ namespace Org.Apache.REEF.Wake.Remote.Impl _codec = new RemoteEventCodec<T>(codec); _cachedClients = new Dictionary<IPEndPoint, ProxyObserver>(); - LocalEndpoint = new IPEndPoint(NetworkUtils.LocalIPAddress, 0); + LocalEndpoint = new IPEndPoint(localAddressProvider.LocalAddress, 0); Identifier = new SocketRemoteIdentifier(LocalEndpoint); } } http://git-wip-us.apache.org/repos/asf/reef/blob/445b85d7/lang/cs/Org.Apache.REEF.Wake/Remote/Impl/DefaultRemoteManagerFactory.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Wake/Remote/Impl/DefaultRemoteManagerFactory.cs b/lang/cs/Org.Apache.REEF.Wake/Remote/Impl/DefaultRemoteManagerFactory.cs index 0cb9d70..3b0df29 100644 --- a/lang/cs/Org.Apache.REEF.Wake/Remote/Impl/DefaultRemoteManagerFactory.cs +++ b/lang/cs/Org.Apache.REEF.Wake/Remote/Impl/DefaultRemoteManagerFactory.cs @@ -29,11 +29,16 @@ namespace Org.Apache.REEF.Wake.Impl { private readonly ITcpPortProvider _tcpPortProvider; private readonly ITcpClientConnectionFactory _tcpClientFactory; + private readonly ILocalAddressProvider _localAddressProvider; [Inject] - private DefaultRemoteManagerFactory(ITcpPortProvider tcpPortProvider, ITcpClientConnectionFactory tcpClientFactory) + private DefaultRemoteManagerFactory( + ITcpPortProvider tcpPortProvider, + ITcpClientConnectionFactory tcpClientFactory, + ILocalAddressProvider localAddressProvider) { _tcpPortProvider = tcpPortProvider; + _localAddressProvider = localAddressProvider; _tcpClientFactory = tcpClientFactory; } @@ -70,7 +75,7 @@ namespace Org.Apache.REEF.Wake.Impl /// <returns>IRemoteManager instance</returns> public IRemoteManager<T> GetInstance<T>(ICodec<T> codec) { - return new DefaultRemoteManager<T>(codec, _tcpClientFactory); + return new DefaultRemoteManager<T>(_localAddressProvider, codec, _tcpClientFactory); } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/445b85d7/lang/cs/Org.Apache.REEF.Wake/Remote/Impl/Link.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Wake/Remote/Impl/Link.cs b/lang/cs/Org.Apache.REEF.Wake/Remote/Impl/Link.cs index 8537817..5049e4b 100644 --- a/lang/cs/Org.Apache.REEF.Wake/Remote/Impl/Link.cs +++ b/lang/cs/Org.Apache.REEF.Wake/Remote/Impl/Link.cs @@ -61,7 +61,7 @@ namespace Org.Apache.REEF.Wake.Remote.Impl _codec = codec; _channel = new Channel(Client.GetStream()); - _localEndpoint = GetLocalEndpoint(); + _localEndpoint = (IPEndPoint)Client.Client.LocalEndPoint; _disposed = false; } @@ -85,7 +85,7 @@ namespace Org.Apache.REEF.Wake.Remote.Impl Client = client; _codec = codec; _channel = new Channel(Client.GetStream()); - _localEndpoint = GetLocalEndpoint(); + _localEndpoint = (IPEndPoint)Client.Client.LocalEndPoint; _disposed = false; } @@ -236,16 +236,5 @@ namespace Org.Apache.REEF.Wake.Remote.Impl { return RemoteEndpoint.GetHashCode(); } - - /// <summary> - /// Discovers the IPEndpoint for the current machine. - /// </summary> - /// <returns>The local IPEndpoint</returns> - private IPEndPoint GetLocalEndpoint() - { - IPAddress address = NetworkUtils.LocalIPAddress; - int port = ((IPEndPoint)Client.Client.LocalEndPoint).Port; - return new IPEndPoint(address, port); - } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/445b85d7/lang/cs/Org.Apache.REEF.Wake/Remote/Impl/LoopbackLocalAddressProvider.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Wake/Remote/Impl/LoopbackLocalAddressProvider.cs b/lang/cs/Org.Apache.REEF.Wake/Remote/Impl/LoopbackLocalAddressProvider.cs new file mode 100644 index 0000000..02ae689 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Wake/Remote/Impl/LoopbackLocalAddressProvider.cs @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Net; +using Org.Apache.REEF.Tang.Annotations; +using Org.Apache.REEF.Tang.Implementations.Tang; +using Org.Apache.REEF.Tang.Interface; +using Org.Apache.REEF.Tang.Util; + +namespace Org.Apache.REEF.Wake.Remote.Impl +{ + /// <summary> + /// Provides the loopback address as the local address. + /// </summary> + public sealed class LoopbackLocalAddressProvider : ILocalAddressProvider, IConfigurationProvider + { + private readonly IConfiguration _configuration; + + [Inject] + private LoopbackLocalAddressProvider() + { + _configuration = TangFactory.GetTang().NewConfigurationBuilder() + .BindImplementation(GenericType<ILocalAddressProvider>.Class, GenericType<LoopbackLocalAddressProvider>.Class) + .Build(); + } + + /// <summary> + /// Returns the loopback address. + /// </summary> + public IPAddress LocalAddress + { + get { return IPAddress.Loopback; } + } + + public IConfiguration GetConfiguration() + { + return _configuration; + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/445b85d7/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/LocalSubmissionFromCS.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/LocalSubmissionFromCS.java b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/LocalSubmissionFromCS.java index bf96d73..3c7e6b2 100644 --- a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/LocalSubmissionFromCS.java +++ b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/LocalSubmissionFromCS.java @@ -35,6 +35,7 @@ import org.apache.reef.runtime.yarn.driver.parameters.JobSubmissionDirectory; import org.apache.reef.tang.Configuration; import org.apache.reef.tang.Configurations; import org.apache.reef.tang.Tang; +import org.apache.reef.wake.remote.address.LoopbackLocalAddressProvider; import org.apache.reef.wake.remote.ports.parameters.TcpPortRangeBegin; import org.apache.reef.wake.remote.ports.parameters.TcpPortRangeCount; import org.apache.reef.wake.remote.ports.parameters.TcpPortRangeTryCount; @@ -108,6 +109,7 @@ final class LocalSubmissionFromCS { final Configuration userProviderConfiguration = Tang.Factory.getTang().newConfigurationBuilder() .bindSetEntry(DriverConfigurationProviders.class, TcpPortConfigurationProvider.class) + .bindSetEntry(DriverConfigurationProviders.class, LoopbackLocalAddressProvider.class) .bindNamedParameter(TcpPortRangeBegin.class, Integer.toString(tcpBeginPort)) .bindNamedParameter(TcpPortRangeCount.class, Integer.toString(tcpRangeCount)) .bindNamedParameter(TcpPortRangeTryCount.class, Integer.toString(tcpTryCount)) http://git-wip-us.apache.org/repos/asf/reef/blob/445b85d7/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/address/LoopbackLocalAddressProvider.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/address/LoopbackLocalAddressProvider.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/address/LoopbackLocalAddressProvider.java new file mode 100644 index 0000000..9baff92 --- /dev/null +++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/remote/address/LoopbackLocalAddressProvider.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.wake.remote.address; + +import org.apache.reef.tang.Configuration; +import org.apache.reef.tang.Tang; +import org.apache.reef.wake.remote.RemoteConfiguration; + +import javax.inject.Inject; +import java.net.InetAddress; + +/** + * A LocalAddressProvider that always uses the Loopback Address. This is used + * mainly in local runtime for C# to prevent firewall message popups. + */ +public final class LoopbackLocalAddressProvider implements LocalAddressProvider { + + @Inject + private LoopbackLocalAddressProvider() { + } + + @Override + public String getLocalAddress() { + // Use the loopback address. + return InetAddress.getLoopbackAddress().getHostAddress(); + } + + @Override + public Configuration getConfiguration() { + return Tang.Factory.getTang().newConfigurationBuilder() + .bind(LocalAddressProvider.class, LoopbackLocalAddressProvider.class) + .bindNamedParameter(RemoteConfiguration.HostAddress.class, getLocalAddress()) + .build(); + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/445b85d7/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpServerImpl.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpServerImpl.java b/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpServerImpl.java index 581cc93..67dce8b 100644 --- a/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpServerImpl.java +++ b/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpServerImpl.java @@ -21,9 +21,12 @@ package org.apache.reef.webserver; import org.apache.reef.tang.annotations.Parameter; import org.apache.reef.util.logging.LoggingScope; import org.apache.reef.util.logging.LoggingScopeFactory; +import org.apache.reef.wake.remote.address.LocalAddressProvider; import org.apache.reef.wake.remote.ports.TcpPortProvider; import org.apache.reef.wake.remote.ports.parameters.TcpPortRangeBegin; +import org.mortbay.jetty.Connector; import org.mortbay.jetty.Server; +import org.mortbay.jetty.bio.SocketConnector; import javax.inject.Inject; import java.net.BindException; @@ -61,6 +64,11 @@ public final class HttpServerImpl implements HttpServer { private final LoggingScopeFactory loggingScopeFactory; /** + * The address provider for the HTTPServer. + */ + private final LocalAddressProvider addressProvider; + + /** * Constructor of HttpServer that wraps Jetty Server. * * @param jettyHandler @@ -69,10 +77,11 @@ public final class HttpServerImpl implements HttpServer { */ @Inject HttpServerImpl(final JettyHandler jettyHandler, - @Parameter(TcpPortRangeBegin.class) final int portNumber, + final LocalAddressProvider addressProvider, + @Parameter(TcpPortRangeBegin.class)final int portNumber, final TcpPortProvider tcpPortProvider, final LoggingScopeFactory loggingScopeFactory) throws Exception { - + this.addressProvider = addressProvider; this.loggingScopeFactory = loggingScopeFactory; this.jettyHandler = jettyHandler; int availablePort = portNumber; @@ -98,7 +107,11 @@ public final class HttpServerImpl implements HttpServer { } private Server tryPort(final int portNumber) throws Exception { - Server srv = new Server(portNumber); + Server srv = new Server(); + final Connector connector = new SocketConnector(); + connector.setHost(addressProvider.getLocalAddress()); + connector.setPort(portNumber); + srv.addConnector(connector); try { srv.start(); LOG.log(Level.INFO, "Jetty Server started with port: {0}", portNumber);
