Repository: reef Updated Branches: refs/heads/master b117b0184 -> 120327fea
[REEF-614] Add logging/tracing level support to JobRequest This addressed the issue by * Modified JobRequest to consider JavaLoggingSetting. * Modified REEF Clients for all environments to consider JavaLoggingSetting. JIRA: [REEF-614](https://issues.apache.org/jira/browse/REEF-614) Pull Request: This closes #946 Project: http://git-wip-us.apache.org/repos/asf/reef/repo Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/120327fe Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/120327fe Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/120327fe Branch: refs/heads/master Commit: 120327fead3b0937756b2c721b13ab702842438b Parents: b117b01 Author: Andrew Chung <[email protected]> Authored: Mon Apr 11 17:10:37 2016 -0700 Committer: Markus Weimer <[email protected]> Committed: Tue Apr 12 16:31:00 2016 -0700 ---------------------------------------------------------------------- .../LegacyJobResourceUploaderTests.cs | 9 +++++++-- lang/cs/Org.Apache.REEF.Client/API/JobParameters.cs | 15 ++++++++++++++- .../API/JobParametersBuilder.cs | 15 ++++++++++++++- lang/cs/Org.Apache.REEF.Client/API/JobRequest.cs | 9 +++++++++ .../Org.Apache.REEF.Client/API/JobRequestBuilder.cs | 7 +++++++ .../Common/IJavaClientLauncher.cs | 3 ++- .../Common/JavaClientLauncher.cs | 14 +++++++++++--- lang/cs/Org.Apache.REEF.Client/Local/LocalClient.cs | 4 ++-- .../YARN/LegacyJobResourceUploader.cs | 5 ++++- .../cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs | 4 +++- .../YARN/YarnCommandProviderConfiguration.cs | 2 ++ .../YARN/YarnREEFDotNetClient.cs | 6 ++++++ .../Logging/JavaLoggingSetting.cs | 13 ++++--------- 13 files changed, 85 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/reef/blob/120327fe/lang/cs/Org.Apache.REEF.Client.Tests/LegacyJobResourceUploaderTests.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client.Tests/LegacyJobResourceUploaderTests.cs b/lang/cs/Org.Apache.REEF.Client.Tests/LegacyJobResourceUploaderTests.cs index b7977b1..323e218 100644 --- a/lang/cs/Org.Apache.REEF.Client.Tests/LegacyJobResourceUploaderTests.cs +++ b/lang/cs/Org.Apache.REEF.Client.Tests/LegacyJobResourceUploaderTests.cs @@ -24,6 +24,7 @@ using Org.Apache.REEF.Client.Common; using Org.Apache.REEF.Client.Yarn; using Org.Apache.REEF.Tang.Implementations.Tang; using Org.Apache.REEF.Tang.Util; +using Org.Apache.REEF.Utilities.Logging; using Xunit; namespace Org.Apache.REEF.Client.Tests @@ -66,7 +67,9 @@ namespace Org.Apache.REEF.Client.Tests // Clientlauncher is called with correct class name, local archive path, upload path and temp file. var notUsedTask = testContext.JavaClientLauncher.Received(1) - .LaunchAsync(javaClassNameForResourceUploader, + .LaunchAsync( + JavaLoggingSetting.Info, + javaClassNameForResourceUploader, anyLocalArchivePath, "ARCHIVE", AnyDriverResourceUploadPath + "/", @@ -77,7 +80,9 @@ namespace Org.Apache.REEF.Client.Tests // Clientlauncher is called with correct class name, local job file path, upload path and temp file. notUsedTask = testContext.JavaClientLauncher.Received(1) - .LaunchAsync(javaClassNameForResourceUploader, + .LaunchAsync( + JavaLoggingSetting.Info, + javaClassNameForResourceUploader, anyLocalJobFilePath, "FILE", AnyDriverResourceUploadPath + "/", http://git-wip-us.apache.org/repos/asf/reef/blob/120327fe/lang/cs/Org.Apache.REEF.Client/API/JobParameters.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/API/JobParameters.cs b/lang/cs/Org.Apache.REEF.Client/API/JobParameters.cs index 0c1d9fd..6d6c307 100644 --- a/lang/cs/Org.Apache.REEF.Client/API/JobParameters.cs +++ b/lang/cs/Org.Apache.REEF.Client/API/JobParameters.cs @@ -16,6 +16,7 @@ // under the License. using Org.Apache.REEF.Utilities; +using Org.Apache.REEF.Utilities.Logging; namespace Org.Apache.REEF.Client.API { @@ -31,13 +32,15 @@ namespace Org.Apache.REEF.Client.API private readonly int _driverMemory; private readonly Optional<string> _stdoutFilePath; private readonly Optional<string> _stderrFilePath; + private readonly JavaLoggingSetting _logSetting; internal JobParameters( string jobIdentifier, int maxApplicationSubmissions, int driverMemory, string stdoutFilePath, - string stderrFilePath) + string stderrFilePath, + JavaLoggingSetting logSetting) { _jobIdentifier = jobIdentifier; _maxApplicationSubmissions = maxApplicationSubmissions; @@ -48,6 +51,8 @@ namespace Org.Apache.REEF.Client.API _stderrFilePath = string.IsNullOrWhiteSpace(stderrFilePath) ? Optional<string>.Empty() : Optional<string>.Of(stderrFilePath); + + _logSetting = logSetting; } /// <summary> @@ -90,5 +95,13 @@ namespace Org.Apache.REEF.Client.API { get { return _stderrFilePath; } } + + /// <summary> + /// Gets the Java log level. + /// </summary> + public JavaLoggingSetting JavaLogLevel + { + get { return _logSetting; } + } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/120327fe/lang/cs/Org.Apache.REEF.Client/API/JobParametersBuilder.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/API/JobParametersBuilder.cs b/lang/cs/Org.Apache.REEF.Client/API/JobParametersBuilder.cs index 9656a0d..06ab7ee 100644 --- a/lang/cs/Org.Apache.REEF.Client/API/JobParametersBuilder.cs +++ b/lang/cs/Org.Apache.REEF.Client/API/JobParametersBuilder.cs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +using Org.Apache.REEF.Utilities.Logging; + namespace Org.Apache.REEF.Client.API { /// <summary> @@ -27,6 +29,7 @@ namespace Org.Apache.REEF.Client.API private int _driverMemory = 512; private string _stdoutFilePath = null; private string _stderrFilePath = null; + private JavaLoggingSetting _javaLogLevel = JavaLoggingSetting.Info; private JobParametersBuilder() { @@ -51,7 +54,8 @@ namespace Org.Apache.REEF.Client.API _maxApplicationSubmissions, _driverMemory, _stdoutFilePath, - _stderrFilePath); + _stderrFilePath, + _javaLogLevel); } /// <summary> @@ -101,5 +105,14 @@ namespace Org.Apache.REEF.Client.API _stderrFilePath = stderrFilePath; return this; } + + /// <summary> + /// Sets the Java Log Level. + /// </summary> + public JobParametersBuilder SetJavaLogLevel(JavaLoggingSetting javaLogLevel) + { + _javaLogLevel = javaLogLevel; + return this; + } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/120327fe/lang/cs/Org.Apache.REEF.Client/API/JobRequest.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/API/JobRequest.cs b/lang/cs/Org.Apache.REEF.Client/API/JobRequest.cs index c20e7d7..9aad57c 100644 --- a/lang/cs/Org.Apache.REEF.Client/API/JobRequest.cs +++ b/lang/cs/Org.Apache.REEF.Client/API/JobRequest.cs @@ -17,6 +17,7 @@ using System.Collections.Generic; using Org.Apache.REEF.Tang.Interface; +using Org.Apache.REEF.Utilities.Logging; namespace Org.Apache.REEF.Client.API { @@ -109,6 +110,14 @@ namespace Org.Apache.REEF.Client.API } /// <summary> + /// Java log level. + /// </summary> + public JavaLoggingSetting JavaLogLevel + { + get { return _jobParameters.JavaLogLevel; } + } + + /// <summary> /// Gets the <see cref="JobParameters"/> for this particular job submission. /// </summary> public JobParameters JobParameters http://git-wip-us.apache.org/repos/asf/reef/blob/120327fe/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilder.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilder.cs b/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilder.cs index be5c0fe..325ed04 100644 --- a/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilder.cs +++ b/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilder.cs @@ -20,6 +20,7 @@ using System.Collections.Generic; using Org.Apache.REEF.Common.Client.Parameters; using Org.Apache.REEF.Tang.Annotations; using Org.Apache.REEF.Tang.Interface; +using Org.Apache.REEF.Utilities.Logging; namespace Org.Apache.REEF.Client.API { @@ -190,5 +191,11 @@ namespace Org.Apache.REEF.Client.API _appParametersBuilder.AddDriverConfigurationProviders(configurationProviders); return this; } + + public JobRequestBuilder SetJavaLogLevel(JavaLoggingSetting javaLogLevel) + { + _jobParametersBuilder.SetJavaLogLevel(javaLogLevel); + return this; + } } } http://git-wip-us.apache.org/repos/asf/reef/blob/120327fe/lang/cs/Org.Apache.REEF.Client/Common/IJavaClientLauncher.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Common/IJavaClientLauncher.cs b/lang/cs/Org.Apache.REEF.Client/Common/IJavaClientLauncher.cs index 0869c15..c6cd784 100644 --- a/lang/cs/Org.Apache.REEF.Client/Common/IJavaClientLauncher.cs +++ b/lang/cs/Org.Apache.REEF.Client/Common/IJavaClientLauncher.cs @@ -18,6 +18,7 @@ using System.Collections.Generic; using System.Threading.Tasks; using Org.Apache.REEF.Tang.Annotations; +using Org.Apache.REEF.Utilities.Logging; namespace Org.Apache.REEF.Client.Common { @@ -27,7 +28,7 @@ namespace Org.Apache.REEF.Client.Common [DefaultImplementation(typeof(JavaClientLauncher))] internal interface IJavaClientLauncher { - Task LaunchAsync(string javaClassName, params string[] parameters); + Task LaunchAsync(JavaLoggingSetting javaLogLevel, string javaClassName, params string[] parameters); /// <summary> /// Add entries to the end of the classpath of the java client. http://git-wip-us.apache.org/repos/asf/reef/blob/120327fe/lang/cs/Org.Apache.REEF.Client/Common/JavaClientLauncher.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Common/JavaClientLauncher.cs b/lang/cs/Org.Apache.REEF.Client/Common/JavaClientLauncher.cs index 8110f8e..2f7e62f 100644 --- a/lang/cs/Org.Apache.REEF.Client/Common/JavaClientLauncher.cs +++ b/lang/cs/Org.Apache.REEF.Client/Common/JavaClientLauncher.cs @@ -50,13 +50,14 @@ namespace Org.Apache.REEF.Client.Common /// <summary> /// Launch a java class in ClientConstants.ClientJarFilePrefix with provided parameters. /// </summary> + /// <param name="javaLogLevel"></param> /// <param name="javaClassName"></param> /// <param name="parameters"></param> - public Task LaunchAsync(string javaClassName, params string[] parameters) + public Task LaunchAsync(JavaLoggingSetting javaLogLevel, string javaClassName, params string[] parameters) { var startInfo = new ProcessStartInfo { - Arguments = AssembleArguments(javaClassName, parameters), + Arguments = AssembleArguments(javaLogLevel, javaClassName, parameters), FileName = GetJavaCommand(), UseShellExecute = false, RedirectStandardOutput = true, @@ -101,12 +102,19 @@ namespace Org.Apache.REEF.Client.Common /// <summary> /// Assembles the command line arguments. Used by LaunchAsync() /// </summary> + /// <param name="javaLogLevel"></param> /// <param name="javaClassName"></param> /// <param name="parameters"></param> /// <returns></returns> - private string AssembleArguments(string javaClassName, params string[] parameters) + private string AssembleArguments(JavaLoggingSetting javaLogLevel, string javaClassName, params string[] parameters) { IList<string> arguments = new List<string>(); + + if (javaLogLevel == JavaLoggingSetting.Verbose) + { + arguments.Add("-Djava.util.logging.config.class=org.apache.reef.util.logging.Config"); + } + arguments.Add("-cp"); arguments.Add(GetClientClasspath()); arguments.Add(javaClassName); http://git-wip-us.apache.org/repos/asf/reef/blob/120327fe/lang/cs/Org.Apache.REEF.Client/Local/LocalClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Local/LocalClient.cs b/lang/cs/Org.Apache.REEF.Client/Local/LocalClient.cs index 6182b11..0f1ef1d 100644 --- a/lang/cs/Org.Apache.REEF.Client/Local/LocalClient.cs +++ b/lang/cs/Org.Apache.REEF.Client/Local/LocalClient.cs @@ -166,7 +166,7 @@ namespace Org.Apache.REEF.Client.Local var driverFolder = PrepareDriverFolder(jobRequest); var submissionJobArgsFilePath = CreateBootstrapAvroJobConfig(jobRequest.JobParameters, driverFolder); var submissionAppArgsFilePath = CreateBootstrapAvroAppConfig(jobRequest.AppParameters, driverFolder); - _javaClientLauncher.LaunchAsync(JavaClassName, submissionJobArgsFilePath, submissionAppArgsFilePath) + _javaClientLauncher.LaunchAsync(jobRequest.JavaLogLevel, JavaClassName, submissionJobArgsFilePath, submissionAppArgsFilePath) .GetAwaiter().GetResult(); Logger.Log(Level.Info, "Submitted the Driver for execution."); } @@ -177,7 +177,7 @@ namespace Org.Apache.REEF.Client.Local var submissionJobArgsFilePath = CreateBootstrapAvroJobConfig(jobRequest.JobParameters, driverFolder); var submissionAppArgsFilePath = CreateBootstrapAvroAppConfig(jobRequest.AppParameters, driverFolder); - _javaClientLauncher.LaunchAsync(JavaClassName, submissionJobArgsFilePath, submissionAppArgsFilePath) + _javaClientLauncher.LaunchAsync(jobRequest.JavaLogLevel, JavaClassName, submissionJobArgsFilePath, submissionAppArgsFilePath) .LogAndIgnoreExceptionIfAny(Logger, "Java launcher failed"); var fileName = Path.Combine(driverFolder, _fileNames.DriverHttpEndpoint); http://git-wip-us.apache.org/repos/asf/reef/blob/120327fe/lang/cs/Org.Apache.REEF.Client/YARN/LegacyJobResourceUploader.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/LegacyJobResourceUploader.cs b/lang/cs/Org.Apache.REEF.Client/YARN/LegacyJobResourceUploader.cs index 9e7d1f0..00610ca 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/LegacyJobResourceUploader.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/LegacyJobResourceUploader.cs @@ -19,6 +19,7 @@ using System; using System.IO; using System.Threading.Tasks; using Org.Apache.REEF.Client.Common; +using Org.Apache.REEF.Client.YARN.Parameters; using Org.Apache.REEF.Client.YARN.RestClient.DataModel; using Org.Apache.REEF.Common.Files; using Org.Apache.REEF.Tang.Annotations; @@ -92,7 +93,9 @@ namespace Org.Apache.REEF.Client.Yarn try { - await _javaLauncher.LaunchAsync(JavaClassNameForResourceUploader, + await _javaLauncher.LaunchAsync( + JavaLoggingSetting.Info, + JavaClassNameForResourceUploader, filePath, resourceType.ToString(), driverUploadPath, http://git-wip-us.apache.org/repos/asf/reef/blob/120327fe/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs b/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs index d22d2ed..8238f2d 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs @@ -24,6 +24,7 @@ using Org.Apache.REEF.Client.API; using Org.Apache.REEF.Client.Common; using Org.Apache.REEF.Client.Yarn.RestClient; using Org.Apache.REEF.Client.YARN; +using Org.Apache.REEF.Client.YARN.Parameters; using Org.Apache.REEF.Client.YARN.RestClient.DataModel; using Org.Apache.REEF.Common.Files; using Org.Apache.REEF.Tang.Annotations; @@ -115,7 +116,8 @@ namespace Org.Apache.REEF.Client.Yarn var submissionAppArgsFilePath = _paramSerializer.SerializeAppFile(jobRequest.AppParameters, paramInjector, driverFolderPath); // Submit the driver - _javaClientLauncher.LaunchAsync(JavaClassName, submissionJobArgsFilePath, submissionAppArgsFilePath) + _javaClientLauncher.LaunchAsync( + jobRequest.JavaLogLevel, JavaClassName, submissionJobArgsFilePath, submissionAppArgsFilePath) .GetAwaiter() .GetResult(); Logger.Log(Level.Info, "Submitted the Driver for execution." + jobRequest.JobIdentifier); http://git-wip-us.apache.org/repos/asf/reef/blob/120327fe/lang/cs/Org.Apache.REEF.Client/YARN/YarnCommandProviderConfiguration.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/YarnCommandProviderConfiguration.cs b/lang/cs/Org.Apache.REEF.Client/YARN/YarnCommandProviderConfiguration.cs index b508074..b112eb7 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/YarnCommandProviderConfiguration.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/YarnCommandProviderConfiguration.cs @@ -25,6 +25,7 @@ namespace Org.Apache.REEF.Client.YARN { public static readonly OptionalParameter<string> DriverStdoutFilePath = new OptionalParameter<string>(); public static readonly OptionalParameter<string> DriverStderrFilePath = new OptionalParameter<string>(); + public static readonly OptionalParameter<bool> JavaDebugLogging = new OptionalParameter<bool>(); public static ConfigurationModule ConfigurationModule { @@ -33,6 +34,7 @@ namespace Org.Apache.REEF.Client.YARN return new YarnCommandProviderConfiguration() .BindNamedParameter(GenericType<DriverStdoutFilePath>.Class, DriverStdoutFilePath) .BindNamedParameter(GenericType<DriverStderrFilePath>.Class, DriverStderrFilePath) + .BindNamedParameter(GenericType<EnableDebugLogging>.Class, JavaDebugLogging) .Build(); } } http://git-wip-us.apache.org/repos/asf/reef/blob/120327fe/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs b/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs index 4edaf13..18f8fbf 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs @@ -153,6 +153,12 @@ namespace Org.Apache.REEF.Client.YARN IReadOnlyCollection<JobResource> jobResources) { var commandProviderConfigModule = YarnCommandProviderConfiguration.ConfigurationModule; + if (jobParameters.JavaLogLevel == JavaLoggingSetting.Verbose) + { + commandProviderConfigModule = commandProviderConfigModule + .Set(YarnCommandProviderConfiguration.JavaDebugLogging, true.ToString().ToLowerInvariant()); + } + if (jobParameters.StdoutFilePath.IsPresent()) { commandProviderConfigModule = commandProviderConfigModule http://git-wip-us.apache.org/repos/asf/reef/blob/120327fe/lang/cs/Org.Apache.REEF.Utilities/Logging/JavaLoggingSetting.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Utilities/Logging/JavaLoggingSetting.cs b/lang/cs/Org.Apache.REEF.Utilities/Logging/JavaLoggingSetting.cs index e907590..fb6ffcc 100644 --- a/lang/cs/Org.Apache.REEF.Utilities/Logging/JavaLoggingSetting.cs +++ b/lang/cs/Org.Apache.REEF.Utilities/Logging/JavaLoggingSetting.cs @@ -20,18 +20,13 @@ namespace Org.Apache.REEF.Utilities.Logging public enum JavaLoggingSetting { /// <summary> - /// info level log, and not transferred to CLR + /// info level log /// </summary> - INFO = 0, + Info = 0, /// <summary> - /// verbose log, but not to CLR + /// verbose log /// </summary> - VERBOSE = 1, - - /// <summary> - /// verbose log, transferred to CLR - /// </summary> - VERBOSE_TO_CLR = 2, + Verbose = 1 } }
