[jira] [Commented] (YARN-4218) Metric for resource*time that was preempted
[ https://issues.apache.org/jira/browse/YARN-4218?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15655459#comment-15655459 ] Hudson commented on YARN-4218: -- SUCCESS: Integrated in Jenkins build Hadoop-trunk-Commit #10815 (See [https://builds.apache.org/job/Hadoop-trunk-Commit/10815/]) YARN-4218. Metric for resource*time that was preempted. Contributed by (epayne: rev 93eeb13164707d0e3556c2bf737bd2ee09a335c6) * (edit) hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java * (edit) hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java * (edit) hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java * (edit) hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java * (edit) hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java * (edit) hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java * (edit) hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java * (edit) hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java * (edit) hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java * (edit) hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java * (edit) hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java * (edit) hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java * (edit) hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java * (edit) hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto * (edit) hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java * (edit) hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java * (edit) hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java * (edit) hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java * (edit) hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java * (edit) hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto * (edit) hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java * (edit) hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java * (edit) hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java * (edit) hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java * (edit) hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java * (edit)
[jira] [Commented] (YARN-4218) Metric for resource*time that was preempted
[ https://issues.apache.org/jira/browse/YARN-4218?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15655289#comment-15655289 ] Eric Payne commented on YARN-4218: -- +1 Thanks [~lichangleo] for the patches and the work done on this JIRA. I will commit this. > Metric for resource*time that was preempted > --- > > Key: YARN-4218 > URL: https://issues.apache.org/jira/browse/YARN-4218 > Project: Hadoop YARN > Issue Type: Bug > Components: resourcemanager >Reporter: Chang Li >Assignee: Chang Li > Attachments: YARN-4218-branch-2.003.patch, YARN-4218.006.patch, > YARN-4218.2.patch, YARN-4218.2.patch, YARN-4218.2.patch, YARN-4218.2.patch, > YARN-4218.3.patch, YARN-4218.4.patch, YARN-4218.5.patch, > YARN-4218.branch-2.2.patch, YARN-4218.branch-2.patch, YARN-4218.patch, > YARN-4218.trunk.2.patch, YARN-4218.trunk.3.patch, YARN-4218.trunk.patch, > YARN-4218.wip.patch, screenshot-1.png, screenshot-2.png, screenshot-3.png > > > After YARN-415 we have the ability to track the resource*time footprint of a > job and preemption metrics shows how many containers were preempted on a job. > However we don't have a metric showing the resource*time footprint cost of > preemption. In other words, we know how many containers were preempted but we > don't have a good measure of how much work was lost as a result of preemption. > We should add this metric so we can analyze how much work preemption is > costing on a grid and better track which jobs were heavily impacted by it. A > job that has 100 containers preempted that only lasted a minute each and were > very small is going to be less impacted than a job that only lost a single > container but that container was huge and had been running for 3 days. -- This message was sent by Atlassian JIRA (v6.3.4#6332) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-4218) Metric for resource*time that was preempted
[
https://issues.apache.org/jira/browse/YARN-4218?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15653752#comment-15653752
]
Hadoop QA commented on YARN-4218:
-
| (x) *{color:red}-1 overall{color}* |
\\
\\
|| Vote || Subsystem || Runtime || Comment ||
| {color:blue}0{color} | {color:blue} reexec {color} | {color:blue} 0m
22s{color} | {color:blue} Docker mode activated. {color} |
| {color:green}+1{color} | {color:green} @author {color} | {color:green} 0m
0s{color} | {color:green} The patch does not contain any @author tags. {color} |
| {color:green}+1{color} | {color:green} test4tests {color} | {color:green} 0m
0s{color} | {color:green} The patch appears to include 10 new or modified test
files. {color} |
| {color:blue}0{color} | {color:blue} mvndep {color} | {color:blue} 0m
35s{color} | {color:blue} Maven dependency ordering for branch {color} |
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 6m
34s{color} | {color:green} branch-2 passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 1m
50s{color} | {color:green} branch-2 passed with JDK v1.8.0_101 {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 2m
12s{color} | {color:green} branch-2 passed with JDK v1.7.0_111 {color} |
| {color:green}+1{color} | {color:green} checkstyle {color} | {color:green} 0m
51s{color} | {color:green} branch-2 passed {color} |
| {color:green}+1{color} | {color:green} mvnsite {color} | {color:green} 2m
53s{color} | {color:green} branch-2 passed {color} |
| {color:green}+1{color} | {color:green} mvneclipse {color} | {color:green} 1m
29s{color} | {color:green} branch-2 passed {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 5m
42s{color} | {color:green} branch-2 passed {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 1m
56s{color} | {color:green} branch-2 passed with JDK v1.8.0_101 {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 2m
8s{color} | {color:green} branch-2 passed with JDK v1.7.0_111 {color} |
| {color:blue}0{color} | {color:blue} mvndep {color} | {color:blue} 0m
10s{color} | {color:blue} Maven dependency ordering for patch {color} |
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 2m
28s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 2m
1s{color} | {color:green} the patch passed with JDK v1.8.0_101 {color} |
| {color:green}+1{color} | {color:green} cc {color} | {color:green} 2m
1s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} javac {color} | {color:green} 2m
1s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 2m
16s{color} | {color:green} the patch passed with JDK v1.7.0_111 {color} |
| {color:green}+1{color} | {color:green} cc {color} | {color:green} 2m
16s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} javac {color} | {color:green} 2m
16s{color} | {color:green} the patch passed {color} |
| {color:orange}-0{color} | {color:orange} checkstyle {color} | {color:orange}
0m 53s{color} | {color:orange} hadoop-yarn-project/hadoop-yarn: The patch
generated 10 new + 922 unchanged - 10 fixed = 932 total (was 932) {color} |
| {color:green}+1{color} | {color:green} mvnsite {color} | {color:green} 2m
49s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} mvneclipse {color} | {color:green} 1m
24s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} whitespace {color} | {color:green} 0m
0s{color} | {color:green} The patch has no whitespace issues. {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 6m
53s{color} | {color:green} the patch passed {color} |
| {color:red}-1{color} | {color:red} javadoc {color} | {color:red} 0m
20s{color} | {color:red}
hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager-jdk1.8.0_101
with JDK v1.8.0_101 generated 1 new + 924 unchanged - 0 fixed = 925 total (was
924) {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 2m
3s{color} | {color:green} the patch passed with JDK v1.7.0_111 {color} |
| {color:green}+1{color} | {color:green} unit {color} | {color:green} 0m
27s{color} | {color:green} hadoop-yarn-api in the patch passed with JDK
v1.7.0_111. {color} |
| {color:green}+1{color} | {color:green} unit {color} | {color:green} 2m
27s{color} | {color:green} hadoop-yarn-common in the patch passed with JDK
v1.7.0_111. {color} |
| {color:green}+1{color} | {color:green} unit {color} | {color:green} 0m
29s{color} | {color:green}
[jira] [Commented] (YARN-4218) Metric for resource*time that was preempted
[
https://issues.apache.org/jira/browse/YARN-4218?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15653631#comment-15653631
]
Hadoop QA commented on YARN-4218:
-
| (x) *{color:red}-1 overall{color}* |
\\
\\
|| Vote || Subsystem || Runtime || Comment ||
| {color:blue}0{color} | {color:blue} reexec {color} | {color:blue} 0m
20s{color} | {color:blue} Docker mode activated. {color} |
| {color:green}+1{color} | {color:green} @author {color} | {color:green} 0m
0s{color} | {color:green} The patch does not contain any @author tags. {color} |
| {color:green}+1{color} | {color:green} test4tests {color} | {color:green} 0m
0s{color} | {color:green} The patch appears to include 11 new or modified test
files. {color} |
| {color:blue}0{color} | {color:blue} mvndep {color} | {color:blue} 0m
10s{color} | {color:blue} Maven dependency ordering for branch {color} |
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 8m
2s{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 6m
29s{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} checkstyle {color} | {color:green} 1m
0s{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} mvnsite {color} | {color:green} 3m
44s{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} mvneclipse {color} | {color:green} 2m
19s{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 5m
43s{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 2m
50s{color} | {color:green} trunk passed {color} |
| {color:blue}0{color} | {color:blue} mvndep {color} | {color:blue} 0m
10s{color} | {color:blue} Maven dependency ordering for patch {color} |
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 2m
24s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 5m
10s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} cc {color} | {color:green} 5m
10s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} javac {color} | {color:green} 5m
10s{color} | {color:green} the patch passed {color} |
| {color:orange}-0{color} | {color:orange} checkstyle {color} | {color:orange}
1m 1s{color} | {color:orange} hadoop-yarn-project/hadoop-yarn: The patch
generated 10 new + 915 unchanged - 10 fixed = 925 total (was 925) {color} |
| {color:green}+1{color} | {color:green} mvnsite {color} | {color:green} 3m
40s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} mvneclipse {color} | {color:green} 2m
14s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} whitespace {color} | {color:green} 0m
0s{color} | {color:green} The patch has no whitespace issues. {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 6m
10s{color} | {color:green} the patch passed {color} |
| {color:red}-1{color} | {color:red} javadoc {color} | {color:red} 0m
31s{color} | {color:red}
hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager
generated 1 new + 924 unchanged - 0 fixed = 925 total (was 924) {color} |
| {color:green}+1{color} | {color:green} unit {color} | {color:green} 0m
35s{color} | {color:green} hadoop-yarn-api in the patch passed. {color} |
| {color:green}+1{color} | {color:green} unit {color} | {color:green} 2m
27s{color} | {color:green} hadoop-yarn-common in the patch passed. {color} |
| {color:green}+1{color} | {color:green} unit {color} | {color:green} 0m
36s{color} | {color:green} hadoop-yarn-server-common in the patch passed.
{color} |
| {color:green}+1{color} | {color:green} unit {color} | {color:green} 3m
22s{color} | {color:green} hadoop-yarn-server-applicationhistoryservice in the
patch passed. {color} |
| {color:green}+1{color} | {color:green} unit {color} | {color:green} 40m
39s{color} | {color:green} hadoop-yarn-server-resourcemanager in the patch
passed. {color} |
| {color:green}+1{color} | {color:green} unit {color} | {color:green} 16m
12s{color} | {color:green} hadoop-yarn-client in the patch passed. {color} |
| {color:green}+1{color} | {color:green} asflicense {color} | {color:green} 0m
37s{color} | {color:green} The patch does not generate ASF License warnings.
{color} |
| {color:black}{color} | {color:black} {color} | {color:black}128m 8s{color} |
{color:black} {color} |
\\
\\
|| Subsystem || Report/Notes ||
| Docker | Image:yetus/hadoop:e809691 |
| JIRA Issue | YARN-4218 |
| JIRA Patch URL |
https://issues.apache.org/jira/secure/attachment/12838311/YARN-4218.006.patch |
|
[jira] [Commented] (YARN-4218) Metric for resource*time that was preempted
[
https://issues.apache.org/jira/browse/YARN-4218?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15625628#comment-15625628
]
Eric Payne commented on YARN-4218:
--
[~lichangleo]. Sorry, I should have been more specific. Can you please fix the
following javadocs errors. These came from the new methods that you created:
{noformat}
[ERROR]
/testptch/hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java:199:
error: @param name not found
[ERROR] * @param memory_seconds the aggregated amount of memory seconds
[ERROR] ^
[ERROR]
/testptch/hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java:203:
warning: no @param for memorySeconds
[ERROR] public abstract void setPreemptedMemorySeconds(long memorySeconds);
[ERROR] ^
[ERROR]
/testptch/hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java:218:
error: @param name not found
[ERROR] * @param vcore_seconds the aggregated number of vcore seconds
[ERROR] ^
[ERROR]
/testptch/hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java:222:
warning: no @param for vcoreSeconds
[ERROR] public abstract void setPreemptedVcoreSeconds(long vcoreSeconds);
[ERROR] ^
{noformat}
> Metric for resource*time that was preempted
> ---
>
> Key: YARN-4218
> URL: https://issues.apache.org/jira/browse/YARN-4218
> Project: Hadoop YARN
> Issue Type: Bug
> Components: resourcemanager
>Reporter: Chang Li
>Assignee: Chang Li
> Attachments: YARN-4218.2.patch, YARN-4218.2.patch, YARN-4218.2.patch,
> YARN-4218.2.patch, YARN-4218.3.patch, YARN-4218.4.patch, YARN-4218.5.patch,
> YARN-4218.branch-2.2.patch, YARN-4218.branch-2.patch, YARN-4218.patch,
> YARN-4218.trunk.2.patch, YARN-4218.trunk.3.patch, YARN-4218.trunk.patch,
> YARN-4218.wip.patch, screenshot-1.png, screenshot-2.png, screenshot-3.png
>
>
> After YARN-415 we have the ability to track the resource*time footprint of a
> job and preemption metrics shows how many containers were preempted on a job.
> However we don't have a metric showing the resource*time footprint cost of
> preemption. In other words, we know how many containers were preempted but we
> don't have a good measure of how much work was lost as a result of preemption.
> We should add this metric so we can analyze how much work preemption is
> costing on a grid and better track which jobs were heavily impacted by it. A
> job that has 100 containers preempted that only lasted a minute each and were
> very small is going to be less impacted than a job that only lost a single
> container but that container was huge and had been running for 3 days.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-4218) Metric for resource*time that was preempted
[ https://issues.apache.org/jira/browse/YARN-4218?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15624458#comment-15624458 ] Chang Li commented on YARN-4218: [~eepayne] hmm, there are tons of errors for protocol javadocs for missing descriptions of why YarnException is thrown and why IOExceptions are thrown. My changes never touches those protocols not sure why it generates those errors for me. To fulfill those thousands of missing javadocs for exceptions or param probably worth a feature... Also I didn't implement those protocols it's hard for me to write correct description... > Metric for resource*time that was preempted > --- > > Key: YARN-4218 > URL: https://issues.apache.org/jira/browse/YARN-4218 > Project: Hadoop YARN > Issue Type: Bug > Components: resourcemanager >Reporter: Chang Li >Assignee: Chang Li > Attachments: YARN-4218.2.patch, YARN-4218.2.patch, YARN-4218.2.patch, > YARN-4218.2.patch, YARN-4218.3.patch, YARN-4218.4.patch, YARN-4218.5.patch, > YARN-4218.branch-2.2.patch, YARN-4218.branch-2.patch, YARN-4218.patch, > YARN-4218.trunk.2.patch, YARN-4218.trunk.3.patch, YARN-4218.trunk.patch, > YARN-4218.wip.patch, screenshot-1.png, screenshot-2.png, screenshot-3.png > > > After YARN-415 we have the ability to track the resource*time footprint of a > job and preemption metrics shows how many containers were preempted on a job. > However we don't have a metric showing the resource*time footprint cost of > preemption. In other words, we know how many containers were preempted but we > don't have a good measure of how much work was lost as a result of preemption. > We should add this metric so we can analyze how much work preemption is > costing on a grid and better track which jobs were heavily impacted by it. A > job that has 100 containers preempted that only lasted a minute each and were > very small is going to be less impacted than a job that only lost a single > container but that container was huge and had been running for 3 days. -- This message was sent by Atlassian JIRA (v6.3.4#6332) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-4218) Metric for resource*time that was preempted
[
https://issues.apache.org/jira/browse/YARN-4218?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15622971#comment-15622971
]
Eric Payne commented on YARN-4218:
--
[~lichangleo], as for the trunk patch, please go ahead and fix the
{{hadoop-yarn-api}} javadoc warnings. I don't think you can fix the
{{hadoop-yarn-server-resourcemanager}} warnings because it's related to the
underscore:
bq. RMAppBlock.java:179: warning: '_' used as an identifier
> Metric for resource*time that was preempted
> ---
>
> Key: YARN-4218
> URL: https://issues.apache.org/jira/browse/YARN-4218
> Project: Hadoop YARN
> Issue Type: Bug
> Components: resourcemanager
>Reporter: Chang Li
>Assignee: Chang Li
> Attachments: YARN-4218.2.patch, YARN-4218.2.patch, YARN-4218.2.patch,
> YARN-4218.2.patch, YARN-4218.3.patch, YARN-4218.4.patch, YARN-4218.5.patch,
> YARN-4218.branch-2.2.patch, YARN-4218.branch-2.patch, YARN-4218.patch,
> YARN-4218.trunk.2.patch, YARN-4218.trunk.3.patch, YARN-4218.trunk.patch,
> YARN-4218.wip.patch, screenshot-1.png, screenshot-2.png, screenshot-3.png
>
>
> After YARN-415 we have the ability to track the resource*time footprint of a
> job and preemption metrics shows how many containers were preempted on a job.
> However we don't have a metric showing the resource*time footprint cost of
> preemption. In other words, we know how many containers were preempted but we
> don't have a good measure of how much work was lost as a result of preemption.
> We should add this metric so we can analyze how much work preemption is
> costing on a grid and better track which jobs were heavily impacted by it. A
> job that has 100 containers preempted that only lasted a minute each and were
> very small is going to be less impacted than a job that only lost a single
> container but that container was huge and had been running for 3 days.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-4218) Metric for resource*time that was preempted
[ https://issues.apache.org/jira/browse/YARN-4218?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15622603#comment-15622603 ] Eric Payne commented on YARN-4218: -- [~lichangleo], sorry one more thing. I also noticed that the latest branch-2 patch (YARN-4218.branch-2.2.patch) did not apply. > Metric for resource*time that was preempted > --- > > Key: YARN-4218 > URL: https://issues.apache.org/jira/browse/YARN-4218 > Project: Hadoop YARN > Issue Type: Bug > Components: resourcemanager >Reporter: Chang Li >Assignee: Chang Li > Attachments: YARN-4218.2.patch, YARN-4218.2.patch, YARN-4218.2.patch, > YARN-4218.2.patch, YARN-4218.3.patch, YARN-4218.4.patch, YARN-4218.5.patch, > YARN-4218.branch-2.2.patch, YARN-4218.branch-2.patch, YARN-4218.patch, > YARN-4218.trunk.2.patch, YARN-4218.trunk.3.patch, YARN-4218.trunk.patch, > YARN-4218.wip.patch, screenshot-1.png, screenshot-2.png, screenshot-3.png > > > After YARN-415 we have the ability to track the resource*time footprint of a > job and preemption metrics shows how many containers were preempted on a job. > However we don't have a metric showing the resource*time footprint cost of > preemption. In other words, we know how many containers were preempted but we > don't have a good measure of how much work was lost as a result of preemption. > We should add this metric so we can analyze how much work preemption is > costing on a grid and better track which jobs were heavily impacted by it. A > job that has 100 containers preempted that only lasted a minute each and were > very small is going to be less impacted than a job that only lost a single > container but that container was huge and had been running for 3 days. -- This message was sent by Atlassian JIRA (v6.3.4#6332) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-4218) Metric for resource*time that was preempted
[
https://issues.apache.org/jira/browse/YARN-4218?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15622586#comment-15622586
]
Eric Payne commented on YARN-4218:
--
Thanks, [~lichangleo], for updating the patches.
I'm looking into the patches, but I would like to provide a quick reminder
about patch naming conventions. From the [Hadoop: How To Contribute
Wiki|https://wiki.apache.org/hadoop/HowToContribute#Generating_a_patch]
{quote}
Patches for trunk should be named according to the Jira, with a version number:
..patch,
{quote}
So, there is no need to put the word {{trunk}} in the name of trunk patches,
and the {{}} is preferred to be {{nnn}}. For example
{{YARN-4218.001.patch}}, {{YARN-4218.002.patch}}, etc.
For non-trunk branches:
{quote}
Patches for a non-trunk branch should be named
-..patch. ... The branch name suffix should
be the exact name of a git branch, such as "branch-2". Jenkins will check the
name of the patch and detect the appropriate branch for testing.
{quote}
The {{branch-2}} patches that you provided are correct except for the preferred
{{}}, but that's not a big deal.
> Metric for resource*time that was preempted
> ---
>
> Key: YARN-4218
> URL: https://issues.apache.org/jira/browse/YARN-4218
> Project: Hadoop YARN
> Issue Type: Bug
> Components: resourcemanager
>Reporter: Chang Li
>Assignee: Chang Li
> Attachments: YARN-4218.2.patch, YARN-4218.2.patch, YARN-4218.2.patch,
> YARN-4218.2.patch, YARN-4218.3.patch, YARN-4218.4.patch, YARN-4218.5.patch,
> YARN-4218.branch-2.2.patch, YARN-4218.branch-2.patch, YARN-4218.patch,
> YARN-4218.trunk.2.patch, YARN-4218.trunk.3.patch, YARN-4218.trunk.patch,
> YARN-4218.wip.patch, screenshot-1.png, screenshot-2.png, screenshot-3.png
>
>
> After YARN-415 we have the ability to track the resource*time footprint of a
> job and preemption metrics shows how many containers were preempted on a job.
> However we don't have a metric showing the resource*time footprint cost of
> preemption. In other words, we know how many containers were preempted but we
> don't have a good measure of how much work was lost as a result of preemption.
> We should add this metric so we can analyze how much work preemption is
> costing on a grid and better track which jobs were heavily impacted by it. A
> job that has 100 containers preempted that only lasted a minute each and were
> very small is going to be less impacted than a job that only lost a single
> container but that container was huge and had been running for 3 days.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-4218) Metric for resource*time that was preempted
[
https://issues.apache.org/jira/browse/YARN-4218?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15621865#comment-15621865
]
Hadoop QA commented on YARN-4218:
-
| (x) *{color:red}-1 overall{color}* |
\\
\\
|| Vote || Subsystem || Runtime || Comment ||
| {color:blue}0{color} | {color:blue} reexec {color} | {color:blue} 0m
23s{color} | {color:blue} Docker mode activated. {color} |
| {color:green}+1{color} | {color:green} @author {color} | {color:green} 0m
0s{color} | {color:green} The patch does not contain any @author tags. {color} |
| {color:green}+1{color} | {color:green} test4tests {color} | {color:green} 0m
0s{color} | {color:green} The patch appears to include 11 new or modified test
files. {color} |
| {color:blue}0{color} | {color:blue} mvndep {color} | {color:blue} 0m
53s{color} | {color:blue} Maven dependency ordering for branch {color} |
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 6m
47s{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 2m
19s{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} checkstyle {color} | {color:green} 0m
49s{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} mvnsite {color} | {color:green} 2m
41s{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} mvneclipse {color} | {color:green} 1m
24s{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 4m
44s{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 1m
57s{color} | {color:green} trunk passed {color} |
| {color:blue}0{color} | {color:blue} mvndep {color} | {color:blue} 0m
10s{color} | {color:blue} Maven dependency ordering for patch {color} |
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 2m
27s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 2m
34s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} cc {color} | {color:green} 2m
34s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} javac {color} | {color:green} 2m
34s{color} | {color:green} the patch passed {color} |
| {color:orange}-0{color} | {color:orange} checkstyle {color} | {color:orange}
0m 51s{color} | {color:orange} hadoop-yarn-project/hadoop-yarn: The patch
generated 10 new + 908 unchanged - 10 fixed = 918 total (was 918) {color} |
| {color:green}+1{color} | {color:green} mvnsite {color} | {color:green} 2m
40s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} mvneclipse {color} | {color:green} 1m
19s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} whitespace {color} | {color:green} 0m
0s{color} | {color:green} The patch has no whitespace issues. {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 6m
21s{color} | {color:green} the patch passed {color} |
| {color:red}-1{color} | {color:red} javadoc {color} | {color:red} 0m
18s{color} | {color:red} hadoop-yarn-api in the patch failed. {color} |
| {color:red}-1{color} | {color:red} javadoc {color} | {color:red} 0m
23s{color} | {color:red}
hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager
generated 1 new + 938 unchanged - 0 fixed = 939 total (was 938) {color} |
| {color:green}+1{color} | {color:green} unit {color} | {color:green} 0m
28s{color} | {color:green} hadoop-yarn-api in the patch passed. {color} |
| {color:green}+1{color} | {color:green} unit {color} | {color:green} 2m
26s{color} | {color:green} hadoop-yarn-common in the patch passed. {color} |
| {color:green}+1{color} | {color:green} unit {color} | {color:green} 0m
33s{color} | {color:green} hadoop-yarn-server-common in the patch passed.
{color} |
| {color:green}+1{color} | {color:green} unit {color} | {color:green} 2m
51s{color} | {color:green} hadoop-yarn-server-applicationhistoryservice in the
patch passed. {color} |
| {color:green}+1{color} | {color:green} unit {color} | {color:green} 36m
5s{color} | {color:green} hadoop-yarn-server-resourcemanager in the patch
passed. {color} |
| {color:green}+1{color} | {color:green} unit {color} | {color:green} 16m
1s{color} | {color:green} hadoop-yarn-client in the patch passed. {color} |
| {color:green}+1{color} | {color:green} asflicense {color} | {color:green} 0m
19s{color} | {color:green} The patch does not generate ASF License warnings.
{color} |
| {color:black}{color} | {color:black} {color} | {color:black}107m 47s{color} |
{color:black} {color} |
\\
\\
|| Subsystem || Report/Notes ||
| Docker |
[jira] [Commented] (YARN-4218) Metric for resource*time that was preempted
[ https://issues.apache.org/jira/browse/YARN-4218?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15621566#comment-15621566 ] Chang Li commented on YARN-4218: were having some troubles to run jobs on trunk, just figured that out.. Will submit a patch for trunk soon > Metric for resource*time that was preempted > --- > > Key: YARN-4218 > URL: https://issues.apache.org/jira/browse/YARN-4218 > Project: Hadoop YARN > Issue Type: Bug > Components: resourcemanager >Reporter: Chang Li >Assignee: Chang Li > Attachments: YARN-4218.2.patch, YARN-4218.2.patch, YARN-4218.2.patch, > YARN-4218.2.patch, YARN-4218.3.patch, YARN-4218.4.patch, YARN-4218.5.patch, > YARN-4218.branch-2.2.patch, YARN-4218.branch-2.patch, YARN-4218.patch, > YARN-4218.trunk.2.patch, YARN-4218.trunk.patch, YARN-4218.wip.patch, > screenshot-1.png, screenshot-2.png, screenshot-3.png > > > After YARN-415 we have the ability to track the resource*time footprint of a > job and preemption metrics shows how many containers were preempted on a job. > However we don't have a metric showing the resource*time footprint cost of > preemption. In other words, we know how many containers were preempted but we > don't have a good measure of how much work was lost as a result of preemption. > We should add this metric so we can analyze how much work preemption is > costing on a grid and better track which jobs were heavily impacted by it. A > job that has 100 containers preempted that only lasted a minute each and were > very small is going to be less impacted than a job that only lost a single > container but that container was huge and had been running for 3 days. -- This message was sent by Atlassian JIRA (v6.3.4#6332) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-4218) Metric for resource*time that was preempted
[
https://issues.apache.org/jira/browse/YARN-4218?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15620790#comment-15620790
]
Hadoop QA commented on YARN-4218:
-
| (x) *{color:red}-1 overall{color}* |
\\
\\
|| Vote || Subsystem || Runtime || Comment ||
| {color:blue}0{color} | {color:blue} reexec {color} | {color:blue} 0m
0s{color} | {color:blue} Docker mode activated. {color} |
| {color:red}-1{color} | {color:red} patch {color} | {color:red} 0m 8s{color}
| {color:red} YARN-4218 does not apply to branch-2.2. Rebase required? Wrong
Branch? See https://wiki.apache.org/hadoop/HowToContribute for help. {color} |
\\
\\
|| Subsystem || Report/Notes ||
| JIRA Issue | YARN-4218 |
| JIRA Patch URL |
https://issues.apache.org/jira/secure/attachment/12836101/YARN-4218.branch-2.2.patch
|
| Console output |
https://builds.apache.org/job/PreCommit-YARN-Build/13689/console |
| Powered by | Apache Yetus 0.4.0-SNAPSHOT http://yetus.apache.org |
This message was automatically generated.
> Metric for resource*time that was preempted
> ---
>
> Key: YARN-4218
> URL: https://issues.apache.org/jira/browse/YARN-4218
> Project: Hadoop YARN
> Issue Type: Bug
> Components: resourcemanager
>Reporter: Chang Li
>Assignee: Chang Li
> Attachments: YARN-4218.2.patch, YARN-4218.2.patch, YARN-4218.2.patch,
> YARN-4218.2.patch, YARN-4218.3.patch, YARN-4218.4.patch, YARN-4218.5.patch,
> YARN-4218.branch-2.2.patch, YARN-4218.branch-2.patch, YARN-4218.patch,
> YARN-4218.trunk.2.patch, YARN-4218.trunk.patch, YARN-4218.wip.patch,
> screenshot-1.png, screenshot-2.png, screenshot-3.png
>
>
> After YARN-415 we have the ability to track the resource*time footprint of a
> job and preemption metrics shows how many containers were preempted on a job.
> However we don't have a metric showing the resource*time footprint cost of
> preemption. In other words, we know how many containers were preempted but we
> don't have a good measure of how much work was lost as a result of preemption.
> We should add this metric so we can analyze how much work preemption is
> costing on a grid and better track which jobs were heavily impacted by it. A
> job that has 100 containers preempted that only lasted a minute each and were
> very small is going to be less impacted than a job that only lost a single
> container but that container was huge and had been running for 3 days.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-4218) Metric for resource*time that was preempted
[
https://issues.apache.org/jira/browse/YARN-4218?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15620299#comment-15620299
]
Hadoop QA commented on YARN-4218:
-
| (x) *{color:red}-1 overall{color}* |
\\
\\
|| Vote || Subsystem || Runtime || Comment ||
| {color:blue}0{color} | {color:blue} reexec {color} | {color:blue} 0m
0s{color} | {color:blue} Docker mode activated. {color} |
| {color:red}-1{color} | {color:red} patch {color} | {color:red} 0m 5s{color}
| {color:red} YARN-4218 does not apply to trunk. Rebase required? Wrong Branch?
See https://wiki.apache.org/hadoop/HowToContribute for help. {color} |
\\
\\
|| Subsystem || Report/Notes ||
| JIRA Issue | YARN-4218 |
| JIRA Patch URL |
https://issues.apache.org/jira/secure/attachment/12836096/YARN-4218.5.patch |
| Console output |
https://builds.apache.org/job/PreCommit-YARN-Build/13684/console |
| Powered by | Apache Yetus 0.4.0-SNAPSHOT http://yetus.apache.org |
This message was automatically generated.
> Metric for resource*time that was preempted
> ---
>
> Key: YARN-4218
> URL: https://issues.apache.org/jira/browse/YARN-4218
> Project: Hadoop YARN
> Issue Type: Bug
> Components: resourcemanager
>Reporter: Chang Li
>Assignee: Chang Li
> Attachments: YARN-4218.2.patch, YARN-4218.2.patch, YARN-4218.2.patch,
> YARN-4218.2.patch, YARN-4218.3.patch, YARN-4218.4.patch, YARN-4218.5.patch,
> YARN-4218.branch-2.patch, YARN-4218.patch, YARN-4218.trunk.2.patch,
> YARN-4218.trunk.patch, YARN-4218.wip.patch, screenshot-1.png,
> screenshot-2.png, screenshot-3.png
>
>
> After YARN-415 we have the ability to track the resource*time footprint of a
> job and preemption metrics shows how many containers were preempted on a job.
> However we don't have a metric showing the resource*time footprint cost of
> preemption. In other words, we know how many containers were preempted but we
> don't have a good measure of how much work was lost as a result of preemption.
> We should add this metric so we can analyze how much work preemption is
> costing on a grid and better track which jobs were heavily impacted by it. A
> job that has 100 containers preempted that only lasted a minute each and were
> very small is going to be less impacted than a job that only lost a single
> container but that container was huge and had been running for 3 days.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-4218) Metric for resource*time that was preempted
[
https://issues.apache.org/jira/browse/YARN-4218?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15619721#comment-15619721
]
Hadoop QA commented on YARN-4218:
-
| (x) *{color:red}-1 overall{color}* |
\\
\\
|| Vote || Subsystem || Runtime || Comment ||
| {color:blue}0{color} | {color:blue} reexec {color} | {color:blue} 0m
22s{color} | {color:blue} Docker mode activated. {color} |
| {color:green}+1{color} | {color:green} @author {color} | {color:green} 0m
0s{color} | {color:green} The patch does not contain any @author tags. {color} |
| {color:green}+1{color} | {color:green} test4tests {color} | {color:green} 0m
0s{color} | {color:green} The patch appears to include 9 new or modified test
files. {color} |
| {color:blue}0{color} | {color:blue} mvndep {color} | {color:blue} 0m
44s{color} | {color:blue} Maven dependency ordering for branch {color} |
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 6m
30s{color} | {color:green} branch-2 passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 1m
52s{color} | {color:green} branch-2 passed with JDK v1.8.0_101 {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 2m
15s{color} | {color:green} branch-2 passed with JDK v1.7.0_111 {color} |
| {color:green}+1{color} | {color:green} checkstyle {color} | {color:green} 0m
52s{color} | {color:green} branch-2 passed {color} |
| {color:green}+1{color} | {color:green} mvnsite {color} | {color:green} 2m
52s{color} | {color:green} branch-2 passed {color} |
| {color:green}+1{color} | {color:green} mvneclipse {color} | {color:green} 1m
28s{color} | {color:green} branch-2 passed {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 5m
38s{color} | {color:green} branch-2 passed {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 1m
50s{color} | {color:green} branch-2 passed with JDK v1.8.0_101 {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 2m
6s{color} | {color:green} branch-2 passed with JDK v1.7.0_111 {color} |
| {color:blue}0{color} | {color:blue} mvndep {color} | {color:blue} 0m
11s{color} | {color:blue} Maven dependency ordering for patch {color} |
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 2m
23s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 1m
50s{color} | {color:green} the patch passed with JDK v1.8.0_101 {color} |
| {color:green}+1{color} | {color:green} cc {color} | {color:green} 1m
50s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} javac {color} | {color:green} 1m
50s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 2m
12s{color} | {color:green} the patch passed with JDK v1.7.0_111 {color} |
| {color:green}+1{color} | {color:green} cc {color} | {color:green} 2m
12s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} javac {color} | {color:green} 2m
12s{color} | {color:green} the patch passed {color} |
| {color:orange}-0{color} | {color:orange} checkstyle {color} | {color:orange}
0m 51s{color} | {color:orange} hadoop-yarn-project/hadoop-yarn: The patch
generated 28 new + 904 unchanged - 8 fixed = 932 total (was 912) {color} |
| {color:green}+1{color} | {color:green} mvnsite {color} | {color:green} 2m
49s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} mvneclipse {color} | {color:green} 1m
24s{color} | {color:green} the patch passed {color} |
| {color:red}-1{color} | {color:red} whitespace {color} | {color:red} 0m
0s{color} | {color:red} The patch has 9 line(s) that end in whitespace. Use git
apply --whitespace=fix <>. Refer https://git-scm.com/docs/git-apply
{color} |
| {color:red}-1{color} | {color:red} whitespace {color} | {color:red} 0m
0s{color} | {color:red} The patch 1 line(s) with tabs. {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 6m
49s{color} | {color:green} the patch passed {color} |
| {color:red}-1{color} | {color:red} javadoc {color} | {color:red} 0m
19s{color} | {color:red}
hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager-jdk1.8.0_101
with JDK v1.8.0_101 generated 1 new + 938 unchanged - 0 fixed = 939 total (was
938) {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 2m
2s{color} | {color:green} the patch passed with JDK v1.7.0_111 {color} |
| {color:green}+1{color} | {color:green} unit {color} | {color:green} 0m
26s{color} | {color:green} hadoop-yarn-api in the patch passed with JDK
v1.7.0_111. {color} |
| {color:green}+1{color} | {color:green} unit {color} | {color:green} 2m
[jira] [Commented] (YARN-4218) Metric for resource*time that was preempted
[
https://issues.apache.org/jira/browse/YARN-4218?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15619512#comment-15619512
]
Hadoop QA commented on YARN-4218:
-
| (x) *{color:red}-1 overall{color}* |
\\
\\
|| Vote || Subsystem || Runtime || Comment ||
| {color:blue}0{color} | {color:blue} reexec {color} | {color:blue} 0m
18s{color} | {color:blue} Docker mode activated. {color} |
| {color:green}+1{color} | {color:green} @author {color} | {color:green} 0m
0s{color} | {color:green} The patch does not contain any @author tags. {color} |
| {color:green}+1{color} | {color:green} test4tests {color} | {color:green} 0m
0s{color} | {color:green} The patch appears to include 9 new or modified test
files. {color} |
| {color:blue}0{color} | {color:blue} mvndep {color} | {color:blue} 0m
9s{color} | {color:blue} Maven dependency ordering for branch {color} |
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 6m
41s{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 2m
18s{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} checkstyle {color} | {color:green} 0m
49s{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} mvnsite {color} | {color:green} 2m
41s{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} mvneclipse {color} | {color:green} 1m
22s{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 4m
27s{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 1m
51s{color} | {color:green} trunk passed {color} |
| {color:blue}0{color} | {color:blue} mvndep {color} | {color:blue} 0m
10s{color} | {color:blue} Maven dependency ordering for patch {color} |
| {color:red}-1{color} | {color:red} mvninstall {color} | {color:red} 0m
30s{color} | {color:red} hadoop-yarn-server-resourcemanager in the patch
failed. {color} |
| {color:red}-1{color} | {color:red} compile {color} | {color:red} 1m
38s{color} | {color:red} hadoop-yarn in the patch failed. {color} |
| {color:red}-1{color} | {color:red} cc {color} | {color:red} 1m 38s{color} |
{color:red} hadoop-yarn in the patch failed. {color} |
| {color:red}-1{color} | {color:red} javac {color} | {color:red} 1m 38s{color}
| {color:red} hadoop-yarn in the patch failed. {color} |
| {color:orange}-0{color} | {color:orange} checkstyle {color} | {color:orange}
0m 48s{color} | {color:orange} hadoop-yarn-project/hadoop-yarn: The patch
generated 27 new + 897 unchanged - 8 fixed = 924 total (was 905) {color} |
| {color:red}-1{color} | {color:red} mvnsite {color} | {color:red} 0m
31s{color} | {color:red} hadoop-yarn-server-resourcemanager in the patch
failed. {color} |
| {color:green}+1{color} | {color:green} mvneclipse {color} | {color:green} 1m
15s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} whitespace {color} | {color:green} 0m
0s{color} | {color:green} The patch has no whitespace issues. {color} |
| {color:red}-1{color} | {color:red} findbugs {color} | {color:red} 0m
21s{color} | {color:red} hadoop-yarn-server-resourcemanager in the patch
failed. {color} |
| {color:red}-1{color} | {color:red} javadoc {color} | {color:red} 0m
20s{color} | {color:red}
hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager
generated 1 new + 938 unchanged - 0 fixed = 939 total (was 938) {color} |
| {color:green}+1{color} | {color:green} unit {color} | {color:green} 0m
25s{color} | {color:green} hadoop-yarn-api in the patch passed. {color} |
| {color:green}+1{color} | {color:green} unit {color} | {color:green} 2m
16s{color} | {color:green} hadoop-yarn-common in the patch passed. {color} |
| {color:green}+1{color} | {color:green} unit {color} | {color:green} 0m
26s{color} | {color:green} hadoop-yarn-server-common in the patch passed.
{color} |
| {color:green}+1{color} | {color:green} unit {color} | {color:green} 3m
8s{color} | {color:green} hadoop-yarn-server-applicationhistoryservice in the
patch passed. {color} |
| {color:red}-1{color} | {color:red} unit {color} | {color:red} 0m 31s{color}
| {color:red} hadoop-yarn-server-resourcemanager in the patch failed. {color} |
| {color:red}-1{color} | {color:red} unit {color} | {color:red} 1m 16s{color}
| {color:red} hadoop-yarn-client in the patch failed. {color} |
| {color:green}+1{color} | {color:green} asflicense {color} | {color:green} 0m
19s{color} | {color:green} The patch does not generate ASF License warnings.
{color} |
| {color:black}{color} | {color:black} {color} | {color:black} 52m 21s{color} |
{color:black} {color} |
\\
\\
|| Reason || Tests ||
| Failed junit tests | hadoop.yarn.client.TestResourceTrackerOnHA |
| |
[jira] [Commented] (YARN-4218) Metric for resource*time that was preempted
[
https://issues.apache.org/jira/browse/YARN-4218?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15619441#comment-15619441
]
Hadoop QA commented on YARN-4218:
-
| (x) *{color:red}-1 overall{color}* |
\\
\\
|| Vote || Subsystem || Runtime || Comment ||
| {color:blue}0{color} | {color:blue} reexec {color} | {color:blue} 0m
17s{color} | {color:blue} Docker mode activated. {color} |
| {color:green}+1{color} | {color:green} @author {color} | {color:green} 0m
0s{color} | {color:green} The patch does not contain any @author tags. {color} |
| {color:green}+1{color} | {color:green} test4tests {color} | {color:green} 0m
0s{color} | {color:green} The patch appears to include 9 new or modified test
files. {color} |
| {color:blue}0{color} | {color:blue} mvndep {color} | {color:blue} 0m
9s{color} | {color:blue} Maven dependency ordering for branch {color} |
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 6m
46s{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 2m
17s{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} checkstyle {color} | {color:green} 0m
49s{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} mvnsite {color} | {color:green} 2m
42s{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} mvneclipse {color} | {color:green} 1m
23s{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 4m
26s{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 1m
49s{color} | {color:green} trunk passed {color} |
| {color:blue}0{color} | {color:blue} mvndep {color} | {color:blue} 0m
10s{color} | {color:blue} Maven dependency ordering for patch {color} |
| {color:red}-1{color} | {color:red} mvninstall {color} | {color:red} 0m
29s{color} | {color:red} hadoop-yarn-server-resourcemanager in the patch
failed. {color} |
| {color:red}-1{color} | {color:red} compile {color} | {color:red} 1m
38s{color} | {color:red} hadoop-yarn in the patch failed. {color} |
| {color:red}-1{color} | {color:red} cc {color} | {color:red} 1m 38s{color} |
{color:red} hadoop-yarn in the patch failed. {color} |
| {color:red}-1{color} | {color:red} javac {color} | {color:red} 1m 38s{color}
| {color:red} hadoop-yarn in the patch failed. {color} |
| {color:orange}-0{color} | {color:orange} checkstyle {color} | {color:orange}
0m 49s{color} | {color:orange} hadoop-yarn-project/hadoop-yarn: The patch
generated 29 new + 896 unchanged - 9 fixed = 925 total (was 905) {color} |
| {color:red}-1{color} | {color:red} mvnsite {color} | {color:red} 0m
30s{color} | {color:red} hadoop-yarn-server-resourcemanager in the patch
failed. {color} |
| {color:green}+1{color} | {color:green} mvneclipse {color} | {color:green} 1m
15s{color} | {color:green} the patch passed {color} |
| {color:red}-1{color} | {color:red} whitespace {color} | {color:red} 0m
0s{color} | {color:red} The patch has 9 line(s) that end in whitespace. Use git
apply --whitespace=fix <>. Refer https://git-scm.com/docs/git-apply
{color} |
| {color:red}-1{color} | {color:red} whitespace {color} | {color:red} 0m
0s{color} | {color:red} The patch 1 line(s) with tabs. {color} |
| {color:red}-1{color} | {color:red} findbugs {color} | {color:red} 0m
21s{color} | {color:red} hadoop-yarn-server-resourcemanager in the patch
failed. {color} |
| {color:red}-1{color} | {color:red} javadoc {color} | {color:red} 0m
20s{color} | {color:red}
hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager
generated 1 new + 938 unchanged - 0 fixed = 939 total (was 938) {color} |
| {color:green}+1{color} | {color:green} unit {color} | {color:green} 0m
24s{color} | {color:green} hadoop-yarn-api in the patch passed. {color} |
| {color:green}+1{color} | {color:green} unit {color} | {color:green} 2m
17s{color} | {color:green} hadoop-yarn-common in the patch passed. {color} |
| {color:green}+1{color} | {color:green} unit {color} | {color:green} 0m
26s{color} | {color:green} hadoop-yarn-server-common in the patch passed.
{color} |
| {color:green}+1{color} | {color:green} unit {color} | {color:green} 3m
9s{color} | {color:green} hadoop-yarn-server-applicationhistoryservice in the
patch passed. {color} |
| {color:red}-1{color} | {color:red} unit {color} | {color:red} 0m 31s{color}
| {color:red} hadoop-yarn-server-resourcemanager in the patch failed. {color} |
| {color:red}-1{color} | {color:red} unit {color} | {color:red} 1m 16s{color}
| {color:red} hadoop-yarn-client in the patch failed. {color} |
| {color:green}+1{color} | {color:green} asflicense {color} | {color:green} 0m
19s{color} | {color:green} The patch does not generate ASF License warnings.
[jira] [Commented] (YARN-4218) Metric for resource*time that was preempted
[
https://issues.apache.org/jira/browse/YARN-4218?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15619394#comment-15619394
]
Chang Li commented on YARN-4218:
it seems patch generated by {code} git diff HEAD --no-prefix {code} is not
accepted by HADOOP QA? post again .trunk patch generated by {code} git diff
trunk {code}
> Metric for resource*time that was preempted
> ---
>
> Key: YARN-4218
> URL: https://issues.apache.org/jira/browse/YARN-4218
> Project: Hadoop YARN
> Issue Type: Bug
> Components: resourcemanager
>Reporter: Chang Li
>Assignee: Chang Li
> Attachments: YARN-4218.2.patch, YARN-4218.2.patch, YARN-4218.2.patch,
> YARN-4218.2.patch, YARN-4218.3.patch, YARN-4218.4.patch, YARN-4218.patch,
> YARN-4218.trunk.patch, YARN-4218.wip.patch, screenshot-1.png,
> screenshot-2.png, screenshot-3.png
>
>
> After YARN-415 we have the ability to track the resource*time footprint of a
> job and preemption metrics shows how many containers were preempted on a job.
> However we don't have a metric showing the resource*time footprint cost of
> preemption. In other words, we know how many containers were preempted but we
> don't have a good measure of how much work was lost as a result of preemption.
> We should add this metric so we can analyze how much work preemption is
> costing on a grid and better track which jobs were heavily impacted by it. A
> job that has 100 containers preempted that only lasted a minute each and were
> very small is going to be less impacted than a job that only lost a single
> container but that container was huge and had been running for 3 days.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-4218) Metric for resource*time that was preempted
[
https://issues.apache.org/jira/browse/YARN-4218?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15619322#comment-15619322
]
Hadoop QA commented on YARN-4218:
-
| (x) *{color:red}-1 overall{color}* |
\\
\\
|| Vote || Subsystem || Runtime || Comment ||
| {color:blue}0{color} | {color:blue} reexec {color} | {color:blue} 0m
0s{color} | {color:blue} Docker mode activated. {color} |
| {color:red}-1{color} | {color:red} patch {color} | {color:red} 0m 5s{color}
| {color:red} YARN-4218 does not apply to trunk. Rebase required? Wrong Branch?
See https://wiki.apache.org/hadoop/HowToContribute for help. {color} |
\\
\\
|| Subsystem || Report/Notes ||
| JIRA Issue | YARN-4218 |
| JIRA Patch URL |
https://issues.apache.org/jira/secure/attachment/12836007/YARN-4218.4.patch |
| Console output |
https://builds.apache.org/job/PreCommit-YARN-Build/13675/console |
| Powered by | Apache Yetus 0.4.0-SNAPSHOT http://yetus.apache.org |
This message was automatically generated.
> Metric for resource*time that was preempted
> ---
>
> Key: YARN-4218
> URL: https://issues.apache.org/jira/browse/YARN-4218
> Project: Hadoop YARN
> Issue Type: Bug
> Components: resourcemanager
>Reporter: Chang Li
>Assignee: Chang Li
> Attachments: YARN-4218.2.patch, YARN-4218.2.patch, YARN-4218.2.patch,
> YARN-4218.2.patch, YARN-4218.3.patch, YARN-4218.4.patch, YARN-4218.patch,
> YARN-4218.wip.patch, screenshot-1.png, screenshot-2.png, screenshot-3.png
>
>
> After YARN-415 we have the ability to track the resource*time footprint of a
> job and preemption metrics shows how many containers were preempted on a job.
> However we don't have a metric showing the resource*time footprint cost of
> preemption. In other words, we know how many containers were preempted but we
> don't have a good measure of how much work was lost as a result of preemption.
> We should add this metric so we can analyze how much work preemption is
> costing on a grid and better track which jobs were heavily impacted by it. A
> job that has 100 containers preempted that only lasted a minute each and were
> very small is going to be less impacted than a job that only lost a single
> container but that container was huge and had been running for 3 days.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-4218) Metric for resource*time that was preempted
[ https://issues.apache.org/jira/browse/YARN-4218?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15616696#comment-15616696 ] Eric Payne commented on YARN-4218: -- [~lichangleo], Thank you very much for adding this useful metric. I'm sorry that it took so long to review this patch. The patch looks good to me. Please upmerge it and I will commit it to trunk. Also, this feature would be good to have in branch-2 and branch-2.8. Can you please also provide a patch for those branches? > Metric for resource*time that was preempted > --- > > Key: YARN-4218 > URL: https://issues.apache.org/jira/browse/YARN-4218 > Project: Hadoop YARN > Issue Type: Bug > Components: resourcemanager >Reporter: Chang Li >Assignee: Chang Li > Attachments: YARN-4218.2.patch, YARN-4218.2.patch, YARN-4218.2.patch, > YARN-4218.2.patch, YARN-4218.3.patch, YARN-4218.patch, YARN-4218.wip.patch, > screenshot-1.png, screenshot-2.png, screenshot-3.png > > > After YARN-415 we have the ability to track the resource*time footprint of a > job and preemption metrics shows how many containers were preempted on a job. > However we don't have a metric showing the resource*time footprint cost of > preemption. In other words, we know how many containers were preempted but we > don't have a good measure of how much work was lost as a result of preemption. > We should add this metric so we can analyze how much work preemption is > costing on a grid and better track which jobs were heavily impacted by it. A > job that has 100 containers preempted that only lasted a minute each and were > very small is going to be less impacted than a job that only lost a single > container but that container was huge and had been running for 3 days. -- This message was sent by Atlassian JIRA (v6.3.4#6332) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-4218) Metric for resource*time that was preempted
[
https://issues.apache.org/jira/browse/YARN-4218?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15116206#comment-15116206
]
Hadoop QA commented on YARN-4218:
-
| (x) *{color:red}-1 overall{color}* |
\\
\\
|| Vote || Subsystem || Runtime || Comment ||
| {color:blue}0{color} | {color:blue} reexec {color} | {color:blue} 0m 0s
{color} | {color:blue} Docker mode activated. {color} |
| {color:green}+1{color} | {color:green} @author {color} | {color:green} 0m 0s
{color} | {color:green} The patch does not contain any @author tags. {color} |
| {color:green}+1{color} | {color:green} test4tests {color} | {color:green} 0m
0s {color} | {color:green} The patch appears to include 9 new or modified test
files. {color} |
| {color:blue}0{color} | {color:blue} mvndep {color} | {color:blue} 1m 9s
{color} | {color:blue} Maven dependency ordering for branch {color} |
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 7m
56s {color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 1m 46s
{color} | {color:green} trunk passed with JDK v1.8.0_66 {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 2m 8s
{color} | {color:green} trunk passed with JDK v1.7.0_91 {color} |
| {color:green}+1{color} | {color:green} checkstyle {color} | {color:green} 0m
36s {color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} mvnsite {color} | {color:green} 2m 48s
{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} mvneclipse {color} | {color:green} 1m
23s {color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 5m
38s {color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 2m 11s
{color} | {color:green} trunk passed with JDK v1.8.0_66 {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 4m 38s
{color} | {color:green} trunk passed with JDK v1.7.0_91 {color} |
| {color:blue}0{color} | {color:blue} mvndep {color} | {color:blue} 0m 54s
{color} | {color:blue} Maven dependency ordering for patch {color} |
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 2m
16s {color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 1m 47s
{color} | {color:green} the patch passed with JDK v1.8.0_66 {color} |
| {color:green}+1{color} | {color:green} cc {color} | {color:green} 1m 47s
{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} javac {color} | {color:green} 1m 47s
{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 2m 5s
{color} | {color:green} the patch passed with JDK v1.7.0_91 {color} |
| {color:green}+1{color} | {color:green} cc {color} | {color:green} 2m 5s
{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} javac {color} | {color:green} 2m 5s
{color} | {color:green} the patch passed {color} |
| {color:red}-1{color} | {color:red} checkstyle {color} | {color:red} 0m 36s
{color} | {color:red} hadoop-yarn-project/hadoop-yarn: patch generated 24 new +
680 unchanged - 7 fixed = 704 total (was 687) {color} |
| {color:green}+1{color} | {color:green} mvnsite {color} | {color:green} 2m 38s
{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} mvneclipse {color} | {color:green} 1m
10s {color} | {color:green} the patch passed {color} |
| {color:red}-1{color} | {color:red} whitespace {color} | {color:red} 0m 0s
{color} | {color:red} The patch has 9 line(s) that end in whitespace. Use git
apply --whitespace=fix. {color} |
| {color:red}-1{color} | {color:red} whitespace {color} | {color:red} 0m 0s
{color} | {color:red} The patch has 1 line(s) with tabs. {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 6m
46s {color} | {color:green} the patch passed {color} |
| {color:red}-1{color} | {color:red} javadoc {color} | {color:red} 2m 59s
{color} | {color:red}
hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager-jdk1.8.0_66
with JDK v1.8.0_66 generated 1 new + 99 unchanged - 1 fixed = 100 total (was
100) {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 2m 0s
{color} | {color:green} the patch passed with JDK v1.8.0_66 {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 4m 27s
{color} | {color:green} the patch passed with JDK v1.7.0_91 {color} |
| {color:green}+1{color} | {color:green} unit {color} | {color:green} 0m 19s
{color} | {color:green} hadoop-yarn-api in the patch passed with JDK v1.8.0_66.
{color} |
| {color:green}+1{color} |
[jira] [Commented] (YARN-4218) Metric for resource*time that was preempted
[
https://issues.apache.org/jira/browse/YARN-4218?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15056649#comment-15056649
]
Hadoop QA commented on YARN-4218:
-
| (x) *{color:red}-1 overall{color}* |
\\
\\
|| Vote || Subsystem || Runtime || Comment ||
| {color:red}-1{color} | {color:red} patch {color} | {color:red} 0m 4s {color}
| {color:red} YARN-4218 does not apply to trunk. Rebase required? Wrong Branch?
See https://wiki.apache.org/hadoop/HowToContribute for help. {color} |
\\
\\
|| Subsystem || Report/Notes ||
| JIRA Patch URL |
https://issues.apache.org/jira/secure/attachment/12777564/YARN-4218.2.patch |
| JIRA Issue | YARN-4218 |
| Powered by | Apache Yetus 0.1.0 http://yetus.apache.org |
| Console output |
https://builds.apache.org/job/PreCommit-YARN-Build/9973/console |
This message was automatically generated.
> Metric for resource*time that was preempted
> ---
>
> Key: YARN-4218
> URL: https://issues.apache.org/jira/browse/YARN-4218
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: Chang Li
>Assignee: Chang Li
> Attachments: YARN-4218.2.patch, YARN-4218.2.patch, YARN-4218.2.patch,
> YARN-4218.2.patch, YARN-4218.patch, YARN-4218.wip.patch, screenshot-1.png,
> screenshot-2.png, screenshot-3.png
>
>
> After YARN-415 we have the ability to track the resource*time footprint of a
> job and preemption metrics shows how many containers were preempted on a job.
> However we don't have a metric showing the resource*time footprint cost of
> preemption. In other words, we know how many containers were preempted but we
> don't have a good measure of how much work was lost as a result of preemption.
> We should add this metric so we can analyze how much work preemption is
> costing on a grid and better track which jobs were heavily impacted by it. A
> job that has 100 containers preempted that only lasted a minute each and were
> very small is going to be less impacted than a job that only lost a single
> container but that container was huge and had been running for 3 days.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (YARN-4218) Metric for resource*time that was preempted
[
https://issues.apache.org/jira/browse/YARN-4218?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=14946090#comment-14946090
]
Hadoop QA commented on YARN-4218:
-
\\
\\
| (x) *{color:red}-1 overall{color}* |
\\
\\
|| Vote || Subsystem || Runtime || Comment ||
| {color:blue}0{color} | pre-patch | 22m 27s | Pre-patch trunk compilation is
healthy. |
| {color:green}+1{color} | @author | 0m 0s | The patch does not contain any
@author tags. |
| {color:green}+1{color} | tests included | 0m 0s | The patch appears to
include 9 new or modified test files. |
| {color:green}+1{color} | javac | 8m 0s | There were no new javac warning
messages. |
| {color:green}+1{color} | javadoc | 10m 22s | There were no new javadoc
warning messages. |
| {color:red}-1{color} | release audit | 0m 15s | The applied patch generated
1 release audit warnings. |
| {color:red}-1{color} | checkstyle | 2m 43s | The applied patch generated 3
new checkstyle issues (total was 10, now 13). |
| {color:red}-1{color} | whitespace | 0m 19s | The patch has 10 line(s) that
end in whitespace. Use git apply --whitespace=fix. |
| {color:green}+1{color} | install | 1m 32s | mvn install still works. |
| {color:green}+1{color} | eclipse:eclipse | 0m 34s | The patch built with
eclipse:eclipse. |
| {color:green}+1{color} | findbugs | 7m 45s | The patch does not introduce
any new Findbugs (version 3.0.0) warnings. |
| {color:green}+1{color} | yarn tests | 0m 24s | Tests passed in
hadoop-yarn-api. |
| {color:green}+1{color} | yarn tests | 6m 59s | Tests passed in
hadoop-yarn-client. |
| {color:green}+1{color} | yarn tests | 2m 3s | Tests passed in
hadoop-yarn-common. |
| {color:green}+1{color} | yarn tests | 3m 17s | Tests passed in
hadoop-yarn-server-applicationhistoryservice. |
| {color:green}+1{color} | yarn tests | 0m 27s | Tests passed in
hadoop-yarn-server-common. |
| {color:red}-1{color} | yarn tests | 17m 29s | Tests failed in
hadoop-yarn-server-resourcemanager. |
| | | 85m 43s | |
\\
\\
|| Reason || Tests ||
| Failed unit tests | hadoop.yarn.server.resourcemanager.TestApplicationCleanup
|
| |
hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesDelegationTokenAuthentication
|
| | hadoop.yarn.server.resourcemanager.TestResourceTrackerService |
| |
hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesAppsModification |
| | hadoop.yarn.server.resourcemanager.TestMoveApplication |
| | hadoop.yarn.server.resourcemanager.TestApplicationMasterService |
| | hadoop.yarn.server.resourcemanager.rmcontainer.TestRMContainerImpl |
| | hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesFairScheduler |
| |
hadoop.yarn.server.resourcemanager.applicationsmanager.TestAMRMRPCNodeUpdates |
| | hadoop.yarn.server.resourcemanager.webapp.TestRMWebServices |
| |
hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacitySchedulerQueueACLs
|
| |
hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesHttpStaticUserPermissions
|
| | hadoop.yarn.server.resourcemanager.monitor.TestSchedulingMonitor |
| | hadoop.yarn.server.resourcemanager.TestRMAdminService |
| | hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesApps |
| | hadoop.yarn.server.resourcemanager.webapp.TestRMWebappAuthentication |
| |
hadoop.yarn.server.resourcemanager.applicationsmanager.TestAMRMRPCResponseId |
| | hadoop.yarn.server.resourcemanager.TestKillApplicationWithRMHA |
| | hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched |
| | hadoop.yarn.server.resourcemanager.security.TestRMDelegationTokens |
| | hadoop.yarn.server.resourcemanager.TestApplicationACLs |
| | hadoop.yarn.server.resourcemanager.scheduler.capacity.TestLeafQueue |
| |
hadoop.yarn.server.resourcemanager.scheduler.capacity.TestWorkPreservingRMRestartForNodeLabel
|
| |
hadoop.yarn.server.resourcemanager.scheduler.capacity.TestApplicationPriority |
| | hadoop.yarn.server.resourcemanager.TestRMHAForNodeLabels |
| | hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesNodeLabels |
| | hadoop.yarn.server.resourcemanager.applicationsmanager.TestAMRestart |
| | hadoop.yarn.server.resourcemanager.TestAMAuthorization |
| | hadoop.yarn.server.resourcemanager.TestRMRestart |
| | hadoop.yarn.server.resourcemanager.recovery.TestZKRMStateStore |
| |
hadoop.yarn.server.resourcemanager.rmapp.attempt.TestRMAppAttemptTransitions |
| |
hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacitySchedulerNodeLabelUpdate
|
| | hadoop.yarn.server.resourcemanager.security.TestClientToAMTokens |
| | hadoop.yarn.server.resourcemanager.TestClientRMService |
| |
hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacityScheduler |
| | hadoop.yarn.server.resourcemanager.resourcetracker.TestNMReconnect |
| | hadoop.yarn.server.resourcemanager.TestSubmitApplicationWithRMHA |
| |
