Author: vinodkv
Date: Thu Apr 25 05:45:15 2013
New Revision: 1475637
URL: http://svn.apache.org/r1475637
Log:
YARN-577. Add application-progress also to ApplicationReport. Contributed by
Hitesh Shah.
MAPREDUCE-5178. Update MR App to set progress in ApplicationReport after
YARN-577. Contributed by Hitesh Shah.
svn merge --ignore-ancestry -c 1475636 ../../trunk/
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1475637&r1=1475636&r2=1475637&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Thu Apr 25
05:45:15 2013
@@ -110,6 +110,9 @@ Release 2.0.5-beta - UNRELEASED
YARN-581. Added a test to verify that app delegation tokens are restored
after RM restart. (Jian He via vinodkv)
+ YARN-577. Add application-progress also to ApplicationReport. (Hitesh Shah
+ via vinodkv)
+
OPTIMIZATIONS
BUG FIXES
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java?rev=1475637&r1=1475636&r2=1475637&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
Thu Apr 25 05:45:15 2013
@@ -253,4 +253,16 @@ public interface ApplicationReport {
@Private
@Unstable
void setApplicationResourceUsageReport(ApplicationResourceUsageReport
appResources);
+
+ /**
+ * Get the application's progress ( range 0.0 to 1.0 )
+ * @return application's progress
+ */
+ @Public
+ @Stable
+ float getProgress();
+
+ @Private
+ @Unstable
+ void setProgress(float progress);
}
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java?rev=1475637&r1=1475636&r2=1475637&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
Thu Apr 25 05:45:15 2013
@@ -214,6 +214,12 @@ implements ApplicationReport {
}
@Override
+ public float getProgress() {
+ ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getProgress();
+ }
+
+ @Override
public void setApplicationId(ApplicationId applicationId) {
maybeInitBuilder();
if (applicationId == null)
@@ -346,6 +352,12 @@ implements ApplicationReport {
}
@Override
+ public void setProgress(float progress) {
+ maybeInitBuilder();
+ builder.setProgress(progress);
+ }
+
+ @Override
public ApplicationReportProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto?rev=1475637&r1=1475636&r2=1475637&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
Thu Apr 25 05:45:15 2013
@@ -162,6 +162,7 @@ message ApplicationReportProto {
optional ApplicationResourceUsageReportProto app_resource_Usage = 16;
optional string originalTrackingUrl = 17;
optional ApplicationAttemptIdProto currentApplicationAttemptId = 18;
+ optional float progress = 19;
}
enum NodeStateProto {
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java?rev=1475637&r1=1475636&r2=1475637&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
Thu Apr 25 05:45:15 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.client.cl
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
+import java.text.DecimalFormat;
import java.util.List;
import org.apache.commons.cli.CommandLine;
@@ -34,7 +35,7 @@ import org.apache.hadoop.yarn.util.Conve
public class ApplicationCLI extends YarnCLI {
private static final String APPLICATIONS_PATTERN =
- "%30s\t%20s\t%10s\t%10s\t%18s\t%18s\t%35s" +
+ "%30s\t%20s\t%10s\t%10s\t%18s\t%18s\t%15s\t%35s" +
System.getProperty("line.separator");
public static void main(String[] args) throws Exception {
@@ -98,12 +99,15 @@ public class ApplicationCLI extends Yarn
writer.println("Total Applications:" + appsReport.size());
writer.printf(APPLICATIONS_PATTERN, "Application-Id",
"Application-Name", "User", "Queue", "State", "Final-State",
- "Tracking-URL");
+ "Progress", "Tracking-URL");
for (ApplicationReport appReport : appsReport) {
+ DecimalFormat formatter = new DecimalFormat("###.##%");
+ String progress = formatter.format(appReport.getProgress());
writer.printf(APPLICATIONS_PATTERN, appReport.getApplicationId(),
appReport.getName(), appReport.getUser(), appReport.getQueue(),
appReport.getYarnApplicationState(), appReport
- .getFinalApplicationStatus(),
appReport.getOriginalTrackingUrl());
+ .getFinalApplicationStatus(),
+ progress, appReport.getOriginalTrackingUrl());
}
writer.flush();
}
@@ -147,6 +151,10 @@ public class ApplicationCLI extends Yarn
appReportStr.println(appReport.getStartTime());
appReportStr.print("\tFinish-Time : ");
appReportStr.println(appReport.getFinishTime());
+ appReportStr.print("\tProgress : ");
+ DecimalFormat formatter = new DecimalFormat("###.##%");
+ String progress = formatter.format(appReport.getProgress());
+ appReportStr.println(progress);
appReportStr.print("\tState : ");
appReportStr.println(appReport.getYarnApplicationState());
appReportStr.print("\tFinal-State : ");
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java?rev=1475637&r1=1475636&r2=1475637&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
Thu Apr 25 05:45:15 2013
@@ -76,7 +76,7 @@ public class TestYarnCLI {
applicationId, BuilderUtils.newApplicationAttemptId(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.SUCCEEDED, null, "N/A");
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f);
when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
newApplicationReport);
int result = cli.run(new String[] { "-status", applicationId.toString() });
@@ -91,6 +91,7 @@ public class TestYarnCLI {
pw.println("\tQueue : queue");
pw.println("\tStart-Time : 0");
pw.println("\tFinish-Time : 0");
+ pw.println("\tProgress : 53.79%");
pw.println("\tState : FINISHED");
pw.println("\tFinal-State : SUCCEEDED");
pw.println("\tTracking-URL : N/A");
@@ -111,7 +112,7 @@ public class TestYarnCLI {
applicationId, BuilderUtils.newApplicationAttemptId(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.SUCCEEDED, null, "N/A");
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f);
List<ApplicationReport> applicationReports = new
ArrayList<ApplicationReport>();
applicationReports.add(newApplicationReport);
when(client.getApplicationList()).thenReturn(applicationReports);
@@ -124,10 +125,12 @@ public class TestYarnCLI {
pw.println("Total Applications:1");
pw.print(" Application-Id\t Application-Name");
pw.print("\t User\t Queue\t State\t ");
- pw.println("Final-State\t Tracking-URL");
+ pw.print("Final-State\t Progress");
+ pw.println("\t Tracking-URL");
pw.print(" application_1234_0005\t ");
pw.print("appname\t user\t queue\t FINISHED\t ");
- pw.println("SUCCEEDED\t N/A");
+ pw.print("SUCCEEDED\t 53.79%");
+ pw.println("\t N/A");
pw.close();
String appsReportStr = baos.toString("UTF-8");
Assert.assertEquals(appsReportStr, sysOutStream.toString());
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java?rev=1475637&r1=1475636&r2=1475637&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java
Thu Apr 25 05:45:15 2013
@@ -333,7 +333,8 @@ public class BuilderUtils {
ClientToken clientToken, YarnApplicationState state, String diagnostics,
String url, long startTime, long finishTime,
FinalApplicationStatus finalStatus,
- ApplicationResourceUsageReport appResources, String origTrackingUrl) {
+ ApplicationResourceUsageReport appResources, String origTrackingUrl,
+ float progress) {
ApplicationReport report = recordFactory
.newRecordInstance(ApplicationReport.class);
report.setApplicationId(applicationId);
@@ -352,6 +353,7 @@ public class BuilderUtils {
report.setFinalApplicationStatus(finalStatus);
report.setApplicationResourceUsageReport(appResources);
report.setOriginalTrackingUrl(origTrackingUrl);
+ report.setProgress(progress);
return report;
}
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java?rev=1475637&r1=1475636&r2=1475637&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
Thu Apr 25 05:45:15 2013
@@ -437,6 +437,7 @@ public class RMAppImpl implements RMApp,
DUMMY_APPLICATION_RESOURCE_USAGE_REPORT;
FinalApplicationStatus finishState = getFinalApplicationStatus();
String diags = UNAVAILABLE;
+ float progress = 0.0f;
if (allowAccess) {
if (this.currentAttempt != null) {
currentApplicationAttemptId = this.currentAttempt.getAppAttemptId();
@@ -446,8 +447,8 @@ public class RMAppImpl implements RMApp,
host = this.currentAttempt.getHost();
rpcPort = this.currentAttempt.getRpcPort();
appUsageReport = currentAttempt.getApplicationResourceUsageReport();
+ progress = currentAttempt.getProgress();
}
-
diags = this.diagnostics.toString();
}
@@ -462,7 +463,7 @@ public class RMAppImpl implements RMApp,
this.name, host, rpcPort, clientToken,
createApplicationState(this.stateMachine.getCurrentState()), diags,
trackingUrl, this.startTime, this.finishTime, finishState,
- appUsageReport, origTrackingUrl);
+ appUsageReport, origTrackingUrl, progress);
} finally {
this.readLock.unlock();
}
