Repository: reef Updated Branches: refs/heads/master 7495f78a2 -> d82ae5c1e
[REEF-227]Add implementation of the REEF Client for HDInsight * Minor fixes for FileSystemJobResourceUploader * Create HDInsightClientConfiguration which provides correct bindings * Add HDI submission to HelloREEF JIRA: [REEF-227](https://issues.apache.org/jira/browse/REEF-227) Pull Request: Closes #850 Project: http://git-wip-us.apache.org/repos/asf/reef/repo Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/d82ae5c1 Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/d82ae5c1 Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/d82ae5c1 Branch: refs/heads/master Commit: d82ae5c1ed92f0bd4f3200fb88777a2806438b9b Parents: 7495f78 Author: Anupam <[email protected]> Authored: Tue Feb 16 13:55:02 2016 -0800 Committer: Andrew Chung <[email protected]> Committed: Mon Feb 22 16:59:24 2016 -0800 ---------------------------------------------------------------------- .../JobResourceUploaderTests.cs | 6 +- .../Org.Apache.REEF.Client.csproj | 8 ++ .../YARN/HDI/HDInsightClientConfiguration.cs | 82 ++++++++++++++++++++ .../YARN/HDI/HDInsightCommandLineEnvironment.cs | 53 +++++++++++++ .../YARN/HDI/HDInsightCredential.cs | 53 +++++++++++++ ...esourceFileRemoteUrlToClusterUrlConverter.cs | 41 ++++++++++ .../YARN/HDI/HDInsightUrlProvider.cs | 42 ++++++++++ .../YARN/HDI/NamedParameters.cs | 42 ++++++++++ ...esourceFileRemoteUrlToClusterUrlConverter.cs | 40 ++++++++++ ...esourceFileRemoteUrlToClusterUrlConverter.cs | 35 +++++++++ .../RESTClient/FileSystemJobResourceUploader.cs | 20 ++--- .../YARN/RESTClient/IUrlProvider.cs | 5 +- .../HelloREEF.cs | 18 ++++- .../Org.Apache.REEF.Examples.HelloREEF.csproj | 6 +- lang/cs/Org.Apache.REEF.sln.DotSettings | 2 + 15 files changed, 437 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/reef/blob/d82ae5c1/lang/cs/Org.Apache.REEF.Client.Tests/JobResourceUploaderTests.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client.Tests/JobResourceUploaderTests.cs b/lang/cs/Org.Apache.REEF.Client.Tests/JobResourceUploaderTests.cs index 5f75d59..cb163f7 100644 --- a/lang/cs/Org.Apache.REEF.Client.Tests/JobResourceUploaderTests.cs +++ b/lang/cs/Org.Apache.REEF.Client.Tests/JobResourceUploaderTests.cs @@ -41,7 +41,7 @@ namespace Org.Apache.REEF.Client.Tests private const string AnyJobFileResourceAbsoluteUri = AnyScheme + AnyHost + AnyJobFileResourcePath; private const string AnyLocalArchivePath = @"Any\Local\Archive\Path.zip"; private const string AnyLocalJobFilePath = @"Any\Local\Folder\Path\job-submission-params.json"; - private const long AnyModificationTime = 1447413621; + private const long AnyModificationTime = 1447413621000; private const long AnyResourceSize = 53092; private static readonly DateTime Epoch = new DateTime(1970, 1, 1, 0, 0, 0, 0); @@ -116,9 +116,9 @@ namespace Org.Apache.REEF.Client.Tests { var injector = TangFactory.GetTang().NewInjector(); FileSystem.GetFileStatus(new Uri(AnyUploadedResourceAbsoluteUri)) - .Returns(new FileStatus(Epoch + TimeSpan.FromSeconds(AnyModificationTime), AnyResourceSize)); + .Returns(new FileStatus(Epoch + TimeSpan.FromMilliseconds(AnyModificationTime), AnyResourceSize)); FileSystem.GetFileStatus(new Uri(AnyJobFileResourceAbsoluteUri)) - .Returns(new FileStatus(Epoch + TimeSpan.FromSeconds(AnyModificationTime), AnyResourceSize)); + .Returns(new FileStatus(Epoch + TimeSpan.FromMilliseconds(AnyModificationTime), AnyResourceSize)); ResourceArchiveFileGenerator.CreateArchiveToUpload(AnyDriverLocalFolderPath) .Returns(AnyLocalArchivePath); FileSystem.CreateUriForPath(AnyDriverResourceUploadPath) http://git-wip-us.apache.org/repos/asf/reef/blob/d82ae5c1/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj index babf263..aefc5fe 100644 --- a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj +++ b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj @@ -99,6 +99,14 @@ under the License. <Compile Include="Local\Parameters\LocalRuntimeDirectory.cs" /> <Compile Include="Local\Parameters\NumberOfEvaluators.cs" /> <Compile Include="Properties\AssemblyInfo.cs" /> + <Compile Include="YARN\HDI\HDInsightClientConfiguration.cs" /> + <Compile Include="YARN\HDI\HDInsightCommandLineEnvironment.cs" /> + <Compile Include="YARN\HDI\HDInsightCredential.cs" /> + <Compile Include="YARN\HDI\HDInsightResourceFileRemoteUrlToClusterUrlConverter.cs" /> + <Compile Include="YARN\HDI\HDInsightUrlProvider.cs" /> + <Compile Include="YARN\HDI\NamedParameters.cs" /> + <Compile Include="YARN\IdentityResourceFileRemoteUrlToClusterUrlConverter.cs" /> + <Compile Include="YARN\IResourceFileRemoteUrlToClusterUrlConverter.cs" /> <Compile Include="YARN\IYarnCommandLineEnvironment.cs" /> <Compile Include="YARN\IYarnJobCommandProvider.cs" /> <Compile Include="YARN\IJobResourceUploader.cs" /> http://git-wip-us.apache.org/repos/asf/reef/blob/d82ae5c1/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightClientConfiguration.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightClientConfiguration.cs b/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightClientConfiguration.cs new file mode 100644 index 0000000..4b91146 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightClientConfiguration.cs @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Security; +using Org.Apache.REEF.Client.API; +using Org.Apache.REEF.Client.Yarn; +using Org.Apache.REEF.Client.Yarn.RestClient; +using Org.Apache.REEF.Client.YARN.Parameters; +using Org.Apache.REEF.Client.YARN.RestClient; +using Org.Apache.REEF.IO.FileSystem.AzureBlob; +using Org.Apache.REEF.Tang.Formats; +using Org.Apache.REEF.Tang.Util; + +namespace Org.Apache.REEF.Client.YARN.HDI +{ + public class HDInsightClientConfiguration : ConfigurationModuleBuilder + { + public static readonly RequiredParameter<string> HDInsightUsernameParameter = new RequiredParameter<string>(); + public static readonly RequiredParameter<string> HDInsightUrlParameter = new RequiredParameter<string>(); + + // Required because user needs to provide the correct path which includes the container name for their + // blob storage account where they want to upload the resource files. This container should be the container that + // HDInsight is using. + public static readonly RequiredParameter<string> JobSubmissionDirectoryPrefix = new RequiredParameter<string>(); + + public static readonly OptionalParameter<string> HDInsightPasswordParameter = new OptionalParameter<string>(); + + public static readonly OptionalParameter<SecureString> HDInsightSecurePasswordParameter = + new OptionalParameter<SecureString>(); + + /// <summary> + /// Provides <see cref="ConfigurationModule"/> to be used to obtain <see cref="IREEFClient"/> to + /// submit REEF application to HDInsight. + /// Sample usage: + /// <code> + /// <![CDATA[ + /// const string connectionString = "ConnString"; + /// var config = HDInsightClientConfiguration.GetConfigurationModule(azureBlockBlobConfig) + /// .Set(AzureBlockBlobFileSystemConfiguration.ConnectionString, connectionString) + /// .Set(HDInsightClientConfiguration.HDInsightUsernameParameter, @"foo") + /// .Set(HDInsightClientConfiguration.HDInsightUrlParameter, @"https://bar.azurehdinsight.net/";) + /// ...; + /// var client = TangFactory.GetTang().NewInjector(config).GetInstance<IREEFClient>(); + /// ]]> + /// </code> + /// </summary> + public static ConfigurationModule ConfigurationModule = new HDInsightClientConfiguration() + .BindNamedParameter(GenericType<HDInsightUrl>.Class, HDInsightUrlParameter) + .BindNamedParameter(GenericType<HDInsightUserName>.Class, HDInsightUsernameParameter) + .BindNamedParameter(GenericType<HDInsightPasswordSecureString>.Class, + HDInsightSecurePasswordParameter) + .BindNamedParameter(GenericType<JobSubmissionDirectoryPrefixParameter>.Class, + JobSubmissionDirectoryPrefix) + .BindNamedParameter(GenericType<HDInsightPasswordString>.Class, HDInsightPasswordParameter) + .BindImplementation(GenericType<IREEFClient>.Class, GenericType<YarnREEFDotNetClient>.Class) + .BindImplementation(GenericType<IYarnRestClientCredential>.Class, + GenericType<HDInsightCredential>.Class) + .BindImplementation(GenericType<IUrlProvider>.Class, GenericType<HDInsightUrlProvider>.Class) + .BindImplementation(GenericType<IJobResourceUploader>.Class, + GenericType<FileSystemJobResourceUploader>.Class) + .BindImplementation(GenericType<IYarnCommandLineEnvironment>.Class, + GenericType<HDInsightCommandLineEnvironment>.Class) + .BindImplementation(GenericType<IResourceFileRemoteUrlToClusterUrlConverter>.Class, + GenericType<HDInsightResourceFileRemoteUrlToClusterUrlConverter>.Class) + .Merge(AzureBlockBlobFileSystemConfiguration.ConfigurationModule) + .Build(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/d82ae5c1/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightCommandLineEnvironment.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightCommandLineEnvironment.cs b/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightCommandLineEnvironment.cs new file mode 100644 index 0000000..e7852c6 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightCommandLineEnvironment.cs @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Collections.Generic; +using Org.Apache.REEF.Client.Yarn; +using Org.Apache.REEF.Tang.Annotations; + +namespace Org.Apache.REEF.Client.YARN.HDI +{ + /// <summary> + /// Provides YARN classpath for HDInsight environment + /// </summary> + public sealed class HDInsightCommandLineEnvironment : IYarnCommandLineEnvironment + { + [Inject] + private HDInsightCommandLineEnvironment() + { + } + + public IReadOnlyList<string> GetYarnClasspathList() + { + return new List<string> + { + "%HADOOP_HOME%/etc/hadoop", + "%HADOOP_HOME%/share/hadoop/common/lib/*", + "%HADOOP_HOME%/share/hadoop/common/*", + "%HADOOP_HOME%/share/hadoop/hdfs", + "%HADOOP_HOME%/share/hadoop/hdfs/lib/*", + "%HADOOP_HOME%/share/hadoop/hdfs/*", + "%HADOOP_HOME%/share/hadoop/yarn/lib/*", + "%HADOOP_HOME%/share/hadoop/yarn/*", + "%HADOOP_HOME%/share/hadoop/mapreduce/lib/*", + "%HADOOP_HOME%/share/hadoop/mapreduce/*", + "%HADOOP_HOME%/share/hadoop/yarn/*", + "%HADOOP_HOME%/share/hadoop/yarn/lib/*" + }; + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/d82ae5c1/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightCredential.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightCredential.cs b/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightCredential.cs new file mode 100644 index 0000000..6f723cf --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightCredential.cs @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Net; +using System.Security; +using Org.Apache.REEF.Client.YARN.RestClient; +using Org.Apache.REEF.Tang.Annotations; + +namespace Org.Apache.REEF.Client.YARN.HDI +{ + /// <summary> + /// Provides a container around ICredential to capture HDInsight credential + /// </summary> + internal sealed class HDInsightCredential : IYarnRestClientCredential + { + private readonly ICredentials _credentials; + + [Inject] + private HDInsightCredential( + [Parameter(typeof(HDInsightUserName))] string userName, + [Parameter(typeof(HDInsightPasswordSecureString))] SecureString password) + { + _credentials = new NetworkCredential(userName, password); + } + + [Inject] + private HDInsightCredential( + [Parameter(typeof(HDInsightUserName))] string userName, + [Parameter(typeof(HDInsightPasswordString))] string password) + { + _credentials = new NetworkCredential(userName, password); + } + + public ICredentials Credentials + { + get { return _credentials; } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/d82ae5c1/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightResourceFileRemoteUrlToClusterUrlConverter.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightResourceFileRemoteUrlToClusterUrlConverter.cs b/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightResourceFileRemoteUrlToClusterUrlConverter.cs new file mode 100644 index 0000000..0332e81 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightResourceFileRemoteUrlToClusterUrlConverter.cs @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using System.Linq; +using Org.Apache.REEF.Tang.Annotations; + +namespace Org.Apache.REEF.Client.YARN.HDI +{ + internal class HDInsightResourceFileRemoteUrlToClusterUrlConverter : IResourceFileRemoteUrlToClusterUrlConverter + { + [Inject] + private HDInsightResourceFileRemoteUrlToClusterUrlConverter() + { + } + + public string ConvertToLocalUrl(Uri uri) + { + // [uri]"https://reefhdi.blob.core.windows.net/reefdev/tmp/foo.json"; + // Authority : reefhdi.blob.core.windows.net + // Segments : {/, reefdev/, tmp/, foo.json} + var container = uri.Segments[1].Trim('/'); + var path = string.Join(string.Empty, uri.Segments.Skip(2)); + return string.Format("wasb://{0}@{1}/{2}", container, uri.Authority, path); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/d82ae5c1/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightUrlProvider.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightUrlProvider.cs b/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightUrlProvider.cs new file mode 100644 index 0000000..d8647ee --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightUrlProvider.cs @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Org.Apache.REEF.Client.Yarn.RestClient; +using Org.Apache.REEF.Tang.Annotations; + +namespace Org.Apache.REEF.Client.YARN.HDI +{ + internal sealed class HDInsightUrlProvider : IUrlProvider + { + private readonly string _url; + + [Inject] + private HDInsightUrlProvider([Parameter(typeof(HDInsightUrl))] string url) + { + _url = url; + } + + public Task<IEnumerable<Uri>> GetUrlAsync() + { + return Task.FromResult(Enumerable.Repeat(new Uri(_url), 1)); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/d82ae5c1/lang/cs/Org.Apache.REEF.Client/YARN/HDI/NamedParameters.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/HDI/NamedParameters.cs b/lang/cs/Org.Apache.REEF.Client/YARN/HDI/NamedParameters.cs new file mode 100644 index 0000000..1f18691 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/YARN/HDI/NamedParameters.cs @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Security; +using Org.Apache.REEF.Tang.Annotations; + +namespace Org.Apache.REEF.Client.YARN.HDI +{ + [NamedParameter("User name for HDInsight cluster.")] + public sealed class HDInsightUserName : Name<string> + { + } + + [NamedParameter("Password for HDInsight cluster.")] + public sealed class HDInsightPasswordString : Name<string> + { + } + + [NamedParameter("Password for HDInsight cluster.")] + public sealed class HDInsightPasswordSecureString : Name<SecureString> + { + } + + [NamedParameter("URL for HDInsight cluster like https://foo.azurehdinsight.net/.";)] + public sealed class HDInsightUrl : Name<string> + { + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/d82ae5c1/lang/cs/Org.Apache.REEF.Client/YARN/IResourceFileRemoteUrlToClusterUrlConverter.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/IResourceFileRemoteUrlToClusterUrlConverter.cs b/lang/cs/Org.Apache.REEF.Client/YARN/IResourceFileRemoteUrlToClusterUrlConverter.cs new file mode 100644 index 0000000..dc60d05 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/YARN/IResourceFileRemoteUrlToClusterUrlConverter.cs @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using Org.Apache.REEF.Tang.Annotations; + +namespace Org.Apache.REEF.Client.YARN +{ + /// <summary> + /// For certain filesystem there might be two different URLs for the same resouce, + /// depending on access using an external endpoint or from within the system. + /// One such example is AzureBlockBlobStorageFileSystem + /// https://reefhdi.blob.core.windows.net/reefdev/tmp/foo.txt is the external URL whereas + /// when accessing this file from within HDInsight the URL is + /// wasb://[email protected]/tmp/foo.txt + /// This creates a problem with Job resource upload where SubmitApplication request to YARN + /// needs to use the local path whereas resource upload uses external URL. + /// This interface allows users to specify logic for the conversion between external URL + /// and local URL + /// </summary> + [DefaultImplementation(typeof(IdentityResourceFileRemoteUrlToClusterUrlConverter))] + internal interface IResourceFileRemoteUrlToClusterUrlConverter + { + string ConvertToLocalUrl(Uri url); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/d82ae5c1/lang/cs/Org.Apache.REEF.Client/YARN/IdentityResourceFileRemoteUrlToClusterUrlConverter.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/IdentityResourceFileRemoteUrlToClusterUrlConverter.cs b/lang/cs/Org.Apache.REEF.Client/YARN/IdentityResourceFileRemoteUrlToClusterUrlConverter.cs new file mode 100644 index 0000000..a1fe2a9 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/YARN/IdentityResourceFileRemoteUrlToClusterUrlConverter.cs @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using Org.Apache.REEF.Tang.Annotations; + +namespace Org.Apache.REEF.Client.YARN +{ + internal class IdentityResourceFileRemoteUrlToClusterUrlConverter : IResourceFileRemoteUrlToClusterUrlConverter + { + [Inject] + private IdentityResourceFileRemoteUrlToClusterUrlConverter() + { + } + + public string ConvertToLocalUrl(Uri url) + { + return url.AbsoluteUri; + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/d82ae5c1/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/FileSystemJobResourceUploader.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/FileSystemJobResourceUploader.cs b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/FileSystemJobResourceUploader.cs index 83e5ff1..8414475 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/FileSystemJobResourceUploader.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/FileSystemJobResourceUploader.cs @@ -16,7 +16,6 @@ // under the License. using System; -using System.Collections.Generic; using System.IO; using Org.Apache.REEF.Client.Common; using Org.Apache.REEF.Client.Yarn; @@ -40,19 +39,22 @@ namespace Org.Apache.REEF.Client.YARN.RestClient private static readonly DateTime Epoch = new DateTime(1970, 1, 1, 0, 0, 0, 0); private readonly IResourceArchiveFileGenerator _resourceArchiveFileGenerator; private readonly IFileSystem _fileSystem; - private readonly REEFFileNames _reefFileNames; private readonly IFile _file; + private readonly REEFFileNames _reefFileNames; + private readonly IResourceFileRemoteUrlToClusterUrlConverter _urlConverter; [Inject] private FileSystemJobResourceUploader( IResourceArchiveFileGenerator resourceArchiveFileGenerator, IFileSystem fileSystem, REEFFileNames reefFileNames, - IFile file) + IFile file, + IResourceFileRemoteUrlToClusterUrlConverter urlConverter) { + _urlConverter = urlConverter; + _reefFileNames = reefFileNames; _fileSystem = fileSystem; _resourceArchiveFileGenerator = resourceArchiveFileGenerator; - _reefFileNames = reefFileNames; _file = file; } @@ -66,7 +68,7 @@ namespace Org.Apache.REEF.Client.YARN.RestClient _fileSystem.CreateDirectory(parentDirectoryUri); var archivePath = _resourceArchiveFileGenerator.CreateArchiveToUpload(driverLocalFolderPath); - return GetJobResource(archivePath, ResourceType.ARCHIVE, driverUploadPath); + return GetJobResource(archivePath, ResourceType.ARCHIVE, driverUploadPath, _reefFileNames.GetReefFolderName()); } public JobResource UploadFileResource(string fileLocalPath, string remoteUploadDirectoryPath) @@ -78,7 +80,7 @@ namespace Org.Apache.REEF.Client.YARN.RestClient return GetJobResource(fileLocalPath, ResourceType.FILE, remoteUploadDirectoryPath); } - private JobResource GetJobResource(string filePath, ResourceType resourceType, string driverUploadPath) + private JobResource GetJobResource(string filePath, ResourceType resourceType, string driverUploadPath, string localizedName = null) { if (!_file.Exists(filePath)) { @@ -97,9 +99,9 @@ namespace Org.Apache.REEF.Client.YARN.RestClient return new JobResource { - Name = Path.GetFileNameWithoutExtension(filePath), + Name = localizedName ?? Path.GetFileName(filePath), LastModificationUnixTimestamp = DateTimeToUnixTimestamp(fileStatus.ModificationTime), - RemoteUploadPath = remoteFileUri.AbsoluteUri, + RemoteUploadPath = _urlConverter.ConvertToLocalUrl(remoteFileUri), ResourceSize = fileStatus.LengthBytes, ResourceType = resourceType }; @@ -107,7 +109,7 @@ namespace Org.Apache.REEF.Client.YARN.RestClient private static long DateTimeToUnixTimestamp(DateTime dateTime) { - return (long)(dateTime - Epoch).TotalSeconds; + return (long)(dateTime - Epoch).TotalMilliseconds; } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/d82ae5c1/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IUrlProvider.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IUrlProvider.cs b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IUrlProvider.cs index dd1e8ab..7fae0c4 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IUrlProvider.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IUrlProvider.cs @@ -26,7 +26,10 @@ namespace Org.Apache.REEF.Client.Yarn.RestClient public interface IUrlProvider { /// <summary> - /// Returns available Yarn resourcemanager web address for the environment + /// Returns list of available Yarn resourcemanager addresses for the environment + /// For environments with single endpoint/single RM this will be list of size 1. + /// For environments with multiple RMs with different endpoints this is list of all + /// RMs. Client will try all endpoints until successful/the list is exhausted. /// </summary> /// <returns></returns> Task<IEnumerable<Uri>> GetUrlAsync(); http://git-wip-us.apache.org/repos/asf/reef/blob/d82ae5c1/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs index 8c63953..575a13f 100644 --- a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs +++ b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs @@ -19,12 +19,13 @@ using System; using Org.Apache.REEF.Client.API; using Org.Apache.REEF.Client.Local; using Org.Apache.REEF.Client.Yarn; +using Org.Apache.REEF.Client.YARN.HDI; using Org.Apache.REEF.Driver; +using Org.Apache.REEF.IO.FileSystem.AzureBlob; using Org.Apache.REEF.Tang.Annotations; using Org.Apache.REEF.Tang.Implementations.Tang; using Org.Apache.REEF.Tang.Interface; using Org.Apache.REEF.Tang.Util; -using Org.Apache.REEF.Utilities.Logging; namespace Org.Apache.REEF.Examples.HelloREEF { @@ -33,10 +34,10 @@ namespace Org.Apache.REEF.Examples.HelloREEF /// </summary> public sealed class HelloREEF { - private static readonly Logger Logger = Logger.GetLogger(typeof(HelloREEF)); private const string Local = "local"; private const string YARN = "yarn"; private const string YARNRest = "yarnrest"; + private const string HDInsight = "hdi"; private readonly IREEFClient _reefClient; private readonly JobRequestBuilder _jobRequestBuilder; @@ -84,6 +85,19 @@ namespace Org.Apache.REEF.Examples.HelloREEF return YARNClientConfiguration.ConfigurationModule.Build(); case YARNRest: return YARNClientConfiguration.ConfigurationModuleYARNRest.Build(); + case HDInsight: + // To run against HDInsight please replace placeholders below, with actual values for + // connection string, container name (available at Azure portal) and HDInsight + // credentials (username and password) + const string connectionString = "ConnString"; + const string continerName = "foo"; + return HDInsightClientConfiguration.ConfigurationModule + .Set(HDInsightClientConfiguration.HDInsightPasswordParameter, @"pwd") + .Set(HDInsightClientConfiguration.HDInsightUsernameParameter, @"foo") + .Set(HDInsightClientConfiguration.HDInsightUrlParameter, @"https://foo.azurehdinsight.net/";) + .Set(HDInsightClientConfiguration.JobSubmissionDirectoryPrefix, string.Format(@"/{0}/tmp", continerName)) + .Set(AzureBlockBlobFileSystemConfiguration.ConnectionString, connectionString) + .Build(); default: throw new Exception("Unknown runtime: " + name); } http://git-wip-us.apache.org/repos/asf/reef/blob/d82ae5c1/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Org.Apache.REEF.Examples.HelloREEF.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Org.Apache.REEF.Examples.HelloREEF.csproj b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Org.Apache.REEF.Examples.HelloREEF.csproj index 825170f..c044163 100644 --- a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Org.Apache.REEF.Examples.HelloREEF.csproj +++ b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Org.Apache.REEF.Examples.HelloREEF.csproj @@ -66,6 +66,10 @@ <Project>{cdfb3464-4041-42b1-9271-83af24cd5008}</Project> <Name>Org.Apache.REEF.Wake</Name> </ProjectReference> + <ProjectReference Include="..\Org.Apache.REEF.IO\Org.Apache.REEF.IO.csproj"> + <Project>{DEC0F0A8-DBEF-4EBF-B69C-E2369C15ABF1}</Project> + <Name>Org.Apache.REEF.IO</Name> + </ProjectReference> </ItemGroup> <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> <Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" /> @@ -76,4 +80,4 @@ </PropertyGroup> <Error Condition="!Exists('$(SolutionDir)\.nuget\NuGet.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\.nuget\NuGet.targets'))" /> </Target> -</Project> +</Project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/d82ae5c1/lang/cs/Org.Apache.REEF.sln.DotSettings ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.sln.DotSettings b/lang/cs/Org.Apache.REEF.sln.DotSettings index 00610d9..c937155 100644 --- a/lang/cs/Org.Apache.REEF.sln.DotSettings +++ b/lang/cs/Org.Apache.REEF.sln.DotSettings @@ -49,12 +49,14 @@ KIND, either express or implied. See the License for the
 specific language governing permissions and limitations
 under the License.
 </s:String> + <s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=HD/@EntryIndexedValue">HDI</s:String> <s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=MB/@EntryIndexedValue">MB</s:String> <s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=MB/@EntryIndexedValue">GB</s:String> <s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=REEF/@EntryIndexedValue">REEF</s:String> <s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=REST/@EntryIndexedValue">REST</s:String> <s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=RM/@EntryIndexedValue">RM</s:String> <s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=URL/@EntryIndexedValue">URL</s:String> + <s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=YARN/@EntryIndexedValue">YARN</s:String> <s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=YARNNM/@EntryIndexedValue">YARNNM</s:String> <s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=YARNREST/@EntryIndexedValue">YARNREST</s:String> <s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=YARNRM/@EntryIndexedValue">YARNRM</s:String>
