[jira] [Commented] (YARN-11687) Update CGroupsResourceCalculator to track usages using cgroupv2
[
https://issues.apache.org/jira/browse/YARN-11687?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17848680#comment-17848680
]
ASF GitHub Bot commented on YARN-11687:
---
hadoop-yetus commented on PR #6835:
URL: https://github.com/apache/hadoop/pull/6835#issuecomment-2125205257
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 0m 47s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 1s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 1s | | detect-secrets was not available.
|
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to
include 3 new or modified test files. |
_ trunk Compile Tests _ |
| +0 :ok: | mvndep | 14m 33s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 37m 6s | | trunk passed |
| +1 :green_heart: | compile | 8m 9s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | compile | 7m 49s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | checkstyle | 2m 0s | | trunk passed |
| +1 :green_heart: | mvnsite | 1m 48s | | trunk passed |
| +1 :green_heart: | javadoc | 1m 57s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javadoc | 1m 43s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | spotbugs | 3m 23s | | trunk passed |
| +1 :green_heart: | shadedclient | 39m 55s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 30s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 1m 7s | | the patch passed |
| +1 :green_heart: | compile | 7m 28s | | the patch passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javac | 7m 28s | | the patch passed |
| +1 :green_heart: | compile | 7m 25s | | the patch passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | javac | 7m 25s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks
issues. |
| +1 :green_heart: | checkstyle | 1m 57s | | the patch passed |
| +1 :green_heart: | mvnsite | 1m 38s | | the patch passed |
| -1 :x: | javadoc | 0m 46s |
[/results-javadoc-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/9/artifact/out/results-javadoc-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt)
|
hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1
with JDK Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 generated 1 new + 98
unchanged - 1 fixed = 99 total (was 99) |
| +1 :green_heart: | javadoc | 0m 51s | | hadoop-yarn-common in the
patch passed with JDK Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06. |
| +1 :green_heart: | javadoc | 0m 45s | |
hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager-jdkPrivateBuild-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06
with JDK Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 generated 0 new +
195 unchanged - 1 fixed = 195 total (was 196) |
| -1 :x: | spotbugs | 1m 44s |
[/new-spotbugs-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager.html](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/9/artifact/out/new-spotbugs-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager.html)
|
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager
generated 2 new + 0 unchanged - 0 fixed = 2 total (was 0) |
| +1 :green_heart: | shadedclient | 39m 38s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart: | unit | 5m 34s | | hadoop-yarn-common in the patch
passed. |
| +1 :green_heart: | unit | 25m 12s | | hadoop-yarn-server-nodemanager
in the patch passed. |
| +1 :green_heart: | asflicense | 0m 56s | | The patch does not
generate ASF License warnings
[jira] [Commented] (YARN-11687) Update CGroupsResourceCalculator to track usages using cgroupv2
[
https://issues.apache.org/jira/browse/YARN-11687?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17848567#comment-17848567
]
ASF GitHub Bot commented on YARN-11687:
---
hadoop-yetus commented on PR #6835:
URL: https://github.com/apache/hadoop/pull/6835#issuecomment-2124562337
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 0m 50s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 0s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available.
|
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to
include 3 new or modified test files. |
_ trunk Compile Tests _ |
| +0 :ok: | mvndep | 14m 13s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 36m 53s | | trunk passed |
| +1 :green_heart: | compile | 8m 6s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | compile | 7m 23s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | checkstyle | 1m 57s | | trunk passed |
| +1 :green_heart: | mvnsite | 1m 49s | | trunk passed |
| +1 :green_heart: | javadoc | 1m 56s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javadoc | 1m 47s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | spotbugs | 3m 23s | | trunk passed |
| +1 :green_heart: | shadedclient | 39m 33s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 31s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 1m 8s | | the patch passed |
| +1 :green_heart: | compile | 7m 30s | | the patch passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javac | 7m 30s | | the patch passed |
| +1 :green_heart: | compile | 7m 15s | | the patch passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | javac | 7m 15s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks
issues. |
| -0 :warning: | checkstyle | 1m 52s |
[/results-checkstyle-hadoop-yarn-project_hadoop-yarn.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/8/artifact/out/results-checkstyle-hadoop-yarn-project_hadoop-yarn.txt)
| hadoop-yarn-project/hadoop-yarn: The patch generated 4 new + 11 unchanged -
0 fixed = 15 total (was 11) |
| +1 :green_heart: | mvnsite | 1m 37s | | the patch passed |
| -1 :x: | javadoc | 0m 48s |
[/results-javadoc-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/8/artifact/out/results-javadoc-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt)
|
hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1
with JDK Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 generated 1 new + 98
unchanged - 1 fixed = 99 total (was 99) |
| +1 :green_heart: | javadoc | 0m 55s | | hadoop-yarn-common in the
patch passed with JDK Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06. |
| +1 :green_heart: | javadoc | 0m 45s | |
hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager-jdkPrivateBuild-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06
with JDK Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 generated 0 new +
195 unchanged - 1 fixed = 195 total (was 196) |
| -1 :x: | spotbugs | 1m 42s |
[/new-spotbugs-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager.html](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/8/artifact/out/new-spotbugs-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager.html)
|
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager
generated 2 new + 0 unchanged - 0 fixed = 2 total (was 0) |
| +1 :green_heart: | shadedclient | 41m 29s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart
[jira] [Commented] (YARN-11687) Update CGroupsResourceCalculator to track usages using cgroupv2
[ https://issues.apache.org/jira/browse/YARN-11687?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17848494#comment-17848494 ] ASF GitHub Bot commented on YARN-11687: --- K0K0V0K commented on code in PR #6835: URL: https://github.com/apache/hadoop/pull/6835#discussion_r1609460882 ## hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java: ## @@ -18,338 +18,134 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources; -import org.apache.hadoop.classification.VisibleForTesting; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.hadoop.util.CpuTimeTracker; -import org.apache.hadoop.util.Shell; -import org.apache.hadoop.util.SysInfoLinux; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.util.Clock; -import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; -import org.apache.hadoop.yarn.util.SystemClock; - -import java.io.BufferedReader; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStreamReader; -import java.math.BigInteger; -import java.nio.charset.StandardCharsets; -import java.util.function.Function; -import java.util.regex.Matcher; -import java.util.regex.Pattern; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * A cgroups file-system based Resource calculator without the process tree - * features. + * A CGroupV1 file-system based Resource calculator without the process tree features. * - * CGroups has its limitations. It can only be enabled, if both CPU and memory - * cgroups are enabled with yarn.nodemanager.resource.cpu.enabled and - * yarn.nodemanager.resource.memory.enabled respectively. This means that - * memory limits are enforced by default. You can turn this off and keep - * memory reporting only with yarn.nodemanager.resource.memory.enforced. + * Warning!!! + * ResourceCalculatorProcessTree can be used with mapreduce.job.process-tree.class property. + * However, those instances runs in the mapreduce task, and can not access to the + * ResourceHandlerModule, what is only initialised in the NodeManager process not in the container. + * So this implementation will not work with the mapreduce.job.process-tree.class property. Review Comment: Thansk @brumi1024, i rephrased the text, however a waring log would be complicated, cause the mapreduce project dont have NodeManager dependency, and i suspect this is a real edge case configuration attempt, so i did not create a warning message > Update CGroupsResourceCalculator to track usages using cgroupv2 > --- > > Key: YARN-11687 > URL: https://issues.apache.org/jira/browse/YARN-11687 > Project: Hadoop YARN > Issue Type: Sub-task >Reporter: Benjamin Teke >Assignee: Bence Kosztolnik >Priority: Major > Labels: pull-request-available > > [CGroupsResourceCalculator|https://github.com/apache/hadoop/blob/f609460bda0c2bd87dd3580158e549e2f34f14d5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java] > should also be updated to handle the cgroup v2 changes. -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Created] (YARN-11698) Finished containers shouldn't be stored indefinitely in the NM state store
Adam Binford created YARN-11698: --- Summary: Finished containers shouldn't be stored indefinitely in the NM state store Key: YARN-11698 URL: https://issues.apache.org/jira/browse/YARN-11698 Project: Hadoop YARN Issue Type: Improvement Components: nodemanager Affects Versions: 3.4.0 Reporter: Adam Binford https://issues.apache.org/jira/browse/YARN-4771 updated the container tracking in the state store to only remove containers when their application ends, in order to make sure all containers logs get aggregated even during NM restarts. This can lead to a significant number of containers building up in the state store and a lot of things to recover. Since this was purely for making sure logs get aggregated, it could be done smarter that takes into account both rolling log aggregation or not having log aggregation enabled at all. -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11687) Update CGroupsResourceCalculator to track usages using cgroupv2
[
https://issues.apache.org/jira/browse/YARN-11687?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17848318#comment-17848318
]
ASF GitHub Bot commented on YARN-11687:
---
hadoop-yetus commented on PR #6835:
URL: https://github.com/apache/hadoop/pull/6835#issuecomment-2123157037
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 17m 46s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 1s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 1s | | detect-secrets was not available.
|
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to
include 3 new or modified test files. |
_ trunk Compile Tests _ |
| +0 :ok: | mvndep | 14m 13s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 36m 33s | | trunk passed |
| +1 :green_heart: | compile | 10m 48s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | compile | 9m 27s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | checkstyle | 2m 8s | | trunk passed |
| +1 :green_heart: | mvnsite | 1m 54s | | trunk passed |
| +1 :green_heart: | javadoc | 1m 59s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javadoc | 1m 50s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | spotbugs | 3m 54s | | trunk passed |
| +1 :green_heart: | shadedclient | 40m 30s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 30s | | Maven dependency ordering for patch |
| -1 :x: | mvninstall | 0m 25s |
[/patch-mvninstall-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/4/artifact/out/patch-mvninstall-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager.txt)
| hadoop-yarn-server-nodemanager in the patch failed. |
| -1 :x: | compile | 1m 29s |
[/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/4/artifact/out/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt)
| hadoop-yarn in the patch failed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1. |
| -1 :x: | javac | 1m 29s |
[/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/4/artifact/out/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt)
| hadoop-yarn in the patch failed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1. |
| -1 :x: | compile | 1m 15s |
[/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkPrivateBuild-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/4/artifact/out/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkPrivateBuild-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06.txt)
| hadoop-yarn in the patch failed with JDK Private
Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06. |
| -1 :x: | javac | 1m 15s |
[/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkPrivateBuild-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/4/artifact/out/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkPrivateBuild-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06.txt)
| hadoop-yarn in the patch failed with JDK Private
Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06. |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks
issues. |
| -0 :warning: | checkstyle | 1m 42s |
[/results-checkstyle-hadoop-yarn-project_hadoop-yarn.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/4/artifact/out/results-checkstyle-hadoop-yarn-project_hadoop-yarn.txt)
| hadoop-yarn-project/hadoop-yarn: The patch generated 4 new + 12 unchanged -
0 fixed = 16 total (was 12) |
| -1 :x: | mvnsite | 0m 28s |
[/patch-mvnsite-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/4/artifact/out/patch-mvnsite-hadoop-yarn-project_hadoop-yarn_hadoop-yarn
[jira] [Commented] (YARN-11687) Update CGroupsResourceCalculator to track usages using cgroupv2
[
https://issues.apache.org/jira/browse/YARN-11687?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17848317#comment-17848317
]
ASF GitHub Bot commented on YARN-11687:
---
hadoop-yetus commented on PR #6835:
URL: https://github.com/apache/hadoop/pull/6835#issuecomment-2123156777
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 17m 45s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 0s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available.
|
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to
include 3 new or modified test files. |
_ trunk Compile Tests _ |
| +0 :ok: | mvndep | 14m 13s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 36m 35s | | trunk passed |
| +1 :green_heart: | compile | 10m 49s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | compile | 9m 27s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | checkstyle | 2m 5s | | trunk passed |
| +1 :green_heart: | mvnsite | 1m 55s | | trunk passed |
| +1 :green_heart: | javadoc | 1m 59s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javadoc | 1m 49s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | spotbugs | 3m 57s | | trunk passed |
| +1 :green_heart: | shadedclient | 40m 12s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 31s | | Maven dependency ordering for patch |
| -1 :x: | mvninstall | 0m 28s |
[/patch-mvninstall-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/5/artifact/out/patch-mvninstall-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager.txt)
| hadoop-yarn-server-nodemanager in the patch failed. |
| -1 :x: | compile | 1m 33s |
[/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/5/artifact/out/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt)
| hadoop-yarn in the patch failed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1. |
| -1 :x: | javac | 1m 33s |
[/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/5/artifact/out/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt)
| hadoop-yarn in the patch failed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1. |
| -1 :x: | compile | 1m 14s |
[/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkPrivateBuild-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/5/artifact/out/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkPrivateBuild-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06.txt)
| hadoop-yarn in the patch failed with JDK Private
Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06. |
| -1 :x: | javac | 1m 14s |
[/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkPrivateBuild-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/5/artifact/out/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkPrivateBuild-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06.txt)
| hadoop-yarn in the patch failed with JDK Private
Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06. |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks
issues. |
| -0 :warning: | checkstyle | 1m 43s |
[/results-checkstyle-hadoop-yarn-project_hadoop-yarn.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/5/artifact/out/results-checkstyle-hadoop-yarn-project_hadoop-yarn.txt)
| hadoop-yarn-project/hadoop-yarn: The patch generated 4 new + 12 unchanged -
0 fixed = 16 total (was 12) |
| -1 :x: | mvnsite | 0m 30s |
[/patch-mvnsite-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/5/artifact/out/patch-mvnsite-hadoop-yarn-project_hadoop-yarn_hadoop-yarn
[jira] [Commented] (YARN-11687) Update CGroupsResourceCalculator to track usages using cgroupv2
[
https://issues.apache.org/jira/browse/YARN-11687?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17848308#comment-17848308
]
ASF GitHub Bot commented on YARN-11687:
---
hadoop-yetus commented on PR #6835:
URL: https://github.com/apache/hadoop/pull/6835#issuecomment-2123119923
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 1m 16s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 1s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 0s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available.
|
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to
include 3 new or modified test files. |
_ trunk Compile Tests _ |
| +0 :ok: | mvndep | 15m 3s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 34m 21s | | trunk passed |
| +1 :green_heart: | compile | 8m 56s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | compile | 7m 59s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | checkstyle | 2m 3s | | trunk passed |
| +1 :green_heart: | mvnsite | 1m 50s | | trunk passed |
| +1 :green_heart: | javadoc | 1m 54s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javadoc | 1m 44s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | spotbugs | 3m 28s | | trunk passed |
| +1 :green_heart: | shadedclient | 40m 44s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 32s | | Maven dependency ordering for patch |
| -1 :x: | mvninstall | 0m 26s |
[/patch-mvninstall-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/3/artifact/out/patch-mvninstall-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager.txt)
| hadoop-yarn-server-nodemanager in the patch failed. |
| -1 :x: | compile | 1m 24s |
[/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/3/artifact/out/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt)
| hadoop-yarn in the patch failed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1. |
| -1 :x: | javac | 1m 24s |
[/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/3/artifact/out/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt)
| hadoop-yarn in the patch failed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1. |
| -1 :x: | compile | 1m 8s |
[/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkPrivateBuild-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/3/artifact/out/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkPrivateBuild-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06.txt)
| hadoop-yarn in the patch failed with JDK Private
Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06. |
| -1 :x: | javac | 1m 8s |
[/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkPrivateBuild-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/3/artifact/out/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkPrivateBuild-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06.txt)
| hadoop-yarn in the patch failed with JDK Private
Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06. |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks
issues. |
| -0 :warning: | checkstyle | 1m 37s |
[/results-checkstyle-hadoop-yarn-project_hadoop-yarn.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/3/artifact/out/results-checkstyle-hadoop-yarn-project_hadoop-yarn.txt)
| hadoop-yarn-project/hadoop-yarn: The patch generated 4 new + 11 unchanged -
0 fixed = 15 total (was 11) |
| -1 :x: | mvnsite | 0m 28s |
[/patch-mvnsite-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/3/artifact/out/patch-mvnsite-hadoop-yarn-project_hadoop-yarn_hadoop-yarn
[jira] [Commented] (YARN-11687) Update CGroupsResourceCalculator to track usages using cgroupv2
[
https://issues.apache.org/jira/browse/YARN-11687?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17848307#comment-17848307
]
ASF GitHub Bot commented on YARN-11687:
---
hadoop-yetus commented on PR #6835:
URL: https://github.com/apache/hadoop/pull/6835#issuecomment-2123119135
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 0m 45s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 1s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 1s | | detect-secrets was not available.
|
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to
include 3 new or modified test files. |
_ trunk Compile Tests _ |
| +0 :ok: | mvndep | 14m 18s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 36m 23s | | trunk passed |
| +1 :green_heart: | compile | 8m 18s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | compile | 7m 28s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | checkstyle | 2m 2s | | trunk passed |
| +1 :green_heart: | mvnsite | 1m 53s | | trunk passed |
| +1 :green_heart: | javadoc | 2m 0s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javadoc | 1m 53s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | spotbugs | 3m 28s | | trunk passed |
| +1 :green_heart: | shadedclient | 39m 43s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 33s | | Maven dependency ordering for patch |
| -1 :x: | mvninstall | 0m 26s |
[/patch-mvninstall-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/7/artifact/out/patch-mvninstall-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager.txt)
| hadoop-yarn-server-nodemanager in the patch failed. |
| -1 :x: | compile | 1m 24s |
[/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/7/artifact/out/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt)
| hadoop-yarn in the patch failed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1. |
| -1 :x: | javac | 1m 24s |
[/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/7/artifact/out/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt)
| hadoop-yarn in the patch failed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1. |
| -1 :x: | compile | 1m 11s |
[/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkPrivateBuild-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/7/artifact/out/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkPrivateBuild-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06.txt)
| hadoop-yarn in the patch failed with JDK Private
Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06. |
| -1 :x: | javac | 1m 11s |
[/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkPrivateBuild-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/7/artifact/out/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkPrivateBuild-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06.txt)
| hadoop-yarn in the patch failed with JDK Private
Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06. |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks
issues. |
| -0 :warning: | checkstyle | 1m 40s |
[/results-checkstyle-hadoop-yarn-project_hadoop-yarn.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/7/artifact/out/results-checkstyle-hadoop-yarn-project_hadoop-yarn.txt)
| hadoop-yarn-project/hadoop-yarn: The patch generated 4 new + 11 unchanged -
0 fixed = 15 total (was 11) |
| -1 :x: | mvnsite | 0m 29s |
[/patch-mvnsite-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/7/artifact/out/patch-mvnsite-hadoop-yarn-project_hadoop-yarn_hadoop-yarn
[jira] [Commented] (YARN-11687) Update CGroupsResourceCalculator to track usages using cgroupv2
[
https://issues.apache.org/jira/browse/YARN-11687?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17848286#comment-17848286
]
ASF GitHub Bot commented on YARN-11687:
---
hadoop-yetus commented on PR #6835:
URL: https://github.com/apache/hadoop/pull/6835#issuecomment-2123031824
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 0m 23s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 1s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 0s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available.
|
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to
include 3 new or modified test files. |
_ trunk Compile Tests _ |
| +0 :ok: | mvndep | 14m 2s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 24m 2s | | trunk passed |
| +1 :green_heart: | compile | 4m 26s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | compile | 4m 13s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | checkstyle | 1m 6s | | trunk passed |
| +1 :green_heart: | mvnsite | 1m 4s | | trunk passed |
| +1 :green_heart: | javadoc | 1m 21s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javadoc | 1m 14s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | spotbugs | 2m 5s | | trunk passed |
| +1 :green_heart: | shadedclient | 24m 4s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 22s | | Maven dependency ordering for patch |
| -1 :x: | mvninstall | 0m 19s |
[/patch-mvninstall-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/6/artifact/out/patch-mvninstall-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager.txt)
| hadoop-yarn-server-nodemanager in the patch failed. |
| -1 :x: | compile | 0m 43s |
[/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/6/artifact/out/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt)
| hadoop-yarn in the patch failed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1. |
| -1 :x: | javac | 0m 43s |
[/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/6/artifact/out/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt)
| hadoop-yarn in the patch failed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1. |
| -1 :x: | compile | 0m 37s |
[/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkPrivateBuild-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/6/artifact/out/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkPrivateBuild-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06.txt)
| hadoop-yarn in the patch failed with JDK Private
Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06. |
| -1 :x: | javac | 0m 37s |
[/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkPrivateBuild-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/6/artifact/out/patch-compile-hadoop-yarn-project_hadoop-yarn-jdkPrivateBuild-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06.txt)
| hadoop-yarn in the patch failed with JDK Private
Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06. |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks
issues. |
| -0 :warning: | checkstyle | 0m 50s |
[/results-checkstyle-hadoop-yarn-project_hadoop-yarn.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/6/artifact/out/results-checkstyle-hadoop-yarn-project_hadoop-yarn.txt)
| hadoop-yarn-project/hadoop-yarn: The patch generated 4 new + 11 unchanged -
0 fixed = 15 total (was 11) |
| -1 :x: | mvnsite | 0m 19s |
[/patch-mvnsite-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/6/artifact/out/patch-mvnsite-hadoop-yarn-project_hadoop-yarn_hadoop-yarn
[jira] [Commented] (YARN-11687) Update CGroupsResourceCalculator to track usages using cgroupv2
[
https://issues.apache.org/jira/browse/YARN-11687?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17848281#comment-17848281
]
ASF GitHub Bot commented on YARN-11687:
---
hadoop-yetus commented on PR #6835:
URL: https://github.com/apache/hadoop/pull/6835#issuecomment-2123018200
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 18m 32s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 1s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 1s | | detect-secrets was not available.
|
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to
include 3 new or modified test files. |
_ trunk Compile Tests _ |
| +0 :ok: | mvndep | 15m 11s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 36m 10s | | trunk passed |
| +1 :green_heart: | compile | 8m 13s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | compile | 7m 20s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | checkstyle | 1m 58s | | trunk passed |
| +1 :green_heart: | mvnsite | 1m 49s | | trunk passed |
| +1 :green_heart: | javadoc | 1m 58s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javadoc | 1m 50s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | spotbugs | 3m 25s | | trunk passed |
| +1 :green_heart: | shadedclient | 41m 0s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 33s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 1m 7s | | the patch passed |
| +1 :green_heart: | compile | 7m 31s | | the patch passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javac | 7m 31s | | the patch passed |
| +1 :green_heart: | compile | 8m 26s | | the patch passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | javac | 8m 26s | | the patch passed |
| +1 :green_heart: | blanks | 0m 1s | | The patch has no blanks
issues. |
| -0 :warning: | checkstyle | 1m 55s |
[/results-checkstyle-hadoop-yarn-project_hadoop-yarn.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/2/artifact/out/results-checkstyle-hadoop-yarn-project_hadoop-yarn.txt)
| hadoop-yarn-project/hadoop-yarn: The patch generated 4 new + 11 unchanged -
0 fixed = 15 total (was 11) |
| +1 :green_heart: | mvnsite | 1m 38s | | the patch passed |
| -1 :x: | javadoc | 0m 48s |
[/results-javadoc-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/2/artifact/out/results-javadoc-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt)
|
hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1
with JDK Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 generated 1 new + 98
unchanged - 1 fixed = 99 total (was 99) |
| +1 :green_heart: | javadoc | 0m 56s | | hadoop-yarn-common in the
patch passed with JDK Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06. |
| +1 :green_heart: | javadoc | 0m 45s | |
hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager-jdkPrivateBuild-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06
with JDK Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 generated 0 new +
195 unchanged - 1 fixed = 195 total (was 196) |
| -1 :x: | spotbugs | 1m 44s |
[/new-spotbugs-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager.html](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/2/artifact/out/new-spotbugs-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager.html)
|
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager
generated 2 new + 0 unchanged - 0 fixed = 2 total (was 0) |
| +1 :green_heart: | shadedclient | 38m 6s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart
[jira] [Commented] (YARN-11681) Update the cgroup documentation with v2 support
[ https://issues.apache.org/jira/browse/YARN-11681?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17848256#comment-17848256 ] ASF GitHub Bot commented on YARN-11681: --- brumi1024 commented on PR #6834: URL: https://github.com/apache/hadoop/pull/6834#issuecomment-2122916425 Thanks @K0K0V0K @p-szucs for the review, merging to trunk. > Update the cgroup documentation with v2 support > --- > > Key: YARN-11681 > URL: https://issues.apache.org/jira/browse/YARN-11681 > Project: Hadoop YARN > Issue Type: Sub-task >Reporter: Benjamin Teke >Assignee: Benjamin Teke >Priority: Major > Labels: pull-request-available > > Update the related > [documentation|https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/NodeManagerCgroups.html] > with v2 support. -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11681) Update the cgroup documentation with v2 support
[ https://issues.apache.org/jira/browse/YARN-11681?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17848257#comment-17848257 ] ASF GitHub Bot commented on YARN-11681: --- brumi1024 merged PR #6834: URL: https://github.com/apache/hadoop/pull/6834 > Update the cgroup documentation with v2 support > --- > > Key: YARN-11681 > URL: https://issues.apache.org/jira/browse/YARN-11681 > Project: Hadoop YARN > Issue Type: Sub-task >Reporter: Benjamin Teke >Assignee: Benjamin Teke >Priority: Major > Labels: pull-request-available > > Update the related > [documentation|https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/NodeManagerCgroups.html] > with v2 support. -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11681) Update the cgroup documentation with v2 support
[
https://issues.apache.org/jira/browse/YARN-11681?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17848255#comment-17848255
]
ASF GitHub Bot commented on YARN-11681:
---
hadoop-yetus commented on PR #6834:
URL: https://github.com/apache/hadoop/pull/6834#issuecomment-2122910379
:confetti_ball: **+1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 0m 20s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 0s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available.
|
| +0 :ok: | markdownlint | 0m 0s | | markdownlint was not available.
|
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
_ trunk Compile Tests _ |
| +1 :green_heart: | mvninstall | 32m 46s | | trunk passed |
| +1 :green_heart: | mvnsite | 0m 18s | | trunk passed |
| +1 :green_heart: | shadedclient | 52m 14s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +1 :green_heart: | mvninstall | 0m 9s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks
issues. |
| +1 :green_heart: | mvnsite | 0m 13s | | the patch passed |
| +1 :green_heart: | shadedclient | 19m 23s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart: | asflicense | 0m 25s | | The patch does not
generate ASF License warnings. |
| | | 76m 3s | | |
| Subsystem | Report/Notes |
|--:|:-|
| Docker | ClientAPI=1.44 ServerAPI=1.44 base:
https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6834/3/artifact/out/Dockerfile
|
| GITHUB PR | https://github.com/apache/hadoop/pull/6834 |
| Optional Tests | dupname asflicense mvnsite codespell detsecrets
markdownlint |
| uname | Linux c955a0c2f12d 5.15.0-94-generic #104-Ubuntu SMP Tue Jan 9
15:25:40 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / 303cd8fc432b6069d266a351ee04b392b5cd670d |
| Max. process+thread count | 552 (vs. ulimit of 5500) |
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site U:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site |
| Console output |
https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6834/3/console |
| versions | git=2.25.1 maven=3.6.3 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
> Update the cgroup documentation with v2 support
> ---
>
> Key: YARN-11681
> URL: https://issues.apache.org/jira/browse/YARN-11681
> Project: Hadoop YARN
> Issue Type: Sub-task
>Reporter: Benjamin Teke
>Assignee: Benjamin Teke
>Priority: Major
> Labels: pull-request-available
>
> Update the related
> [documentation|https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/NodeManagerCgroups.html]
> with v2 support.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11687) Update CGroupsResourceCalculator to track usages using cgroupv2
[ https://issues.apache.org/jira/browse/YARN-11687?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17848237#comment-17848237 ] ASF GitHub Bot commented on YARN-11687: --- brumi1024 commented on code in PR #6835: URL: https://github.com/apache/hadoop/pull/6835#discussion_r1608437832 ## hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsV2ResourceCalculator.java: ## @@ -0,0 +1,124 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Stream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.commons.lang3.StringUtils; + +/** + * A CGroupV2 file-system based Resource calculator without the process tree features. + * + * Warning!!! + * ResourceCalculatorProcessTree can be used with mapreduce.job.process-tree.class property. + * However, those instances runs in the mapreduce task, and can not access to the + * ResourceHandlerModule, what is only initialised in the NodeManager process not in the container. + * So this implementation will not work with the mapreduce.job.process-tree.class property. + * + * Limitation: CGroups does not have the ability to measure virtual memory usage. + * This includes memory reserved but not used. + * CGroups measures used memory as sa sum of physical memory and swap usage. Review Comment: ```suggestion * Cgroup measures used memory as a sum of the physical memory and swap usage. ``` ## hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java: ## @@ -18,338 +18,134 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources; -import org.apache.hadoop.classification.VisibleForTesting; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.hadoop.util.CpuTimeTracker; -import org.apache.hadoop.util.Shell; -import org.apache.hadoop.util.SysInfoLinux; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.util.Clock; -import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; -import org.apache.hadoop.yarn.util.SystemClock; - -import java.io.BufferedReader; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStreamReader; -import java.math.BigInteger; -import java.nio.charset.StandardCharsets; -import java.util.function.Function; -import java.util.regex.Matcher; -import java.util.regex.Pattern; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * A cgroups file-system based Resource calculator without the process tree - * features. + * A CGroupV1 file-system based Resource calculator without the process tree features. * - * CGroups has its limitations. It can only be enabled, if both CPU and memory - * cgroups are enabled with yarn.nodemanager.resource.cpu.enabled and - * yarn.nodemanager.resource.memory.enabled respectively. This means that - * memory limits are enforced by default. You can turn this off and keep - * memory reporting only with yarn.nodemanager.resource.memory.enforced. + * Warning!!! + * ResourceCalculatorProcessTree can be used with mapreduce.job.process-tree.class property. + * However, those instances runs in the mapreduce task, and can not access to the + * ResourceHandlerModule, what is only initialised in the NodeManager process not in the container. + * So this implementation will not work with the mapreduce.job.process-tree.class prope
[jira] [Commented] (YARN-11681) Update the cgroup documentation with v2 support
[ https://issues.apache.org/jira/browse/YARN-11681?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17848230#comment-17848230 ] ASF GitHub Bot commented on YARN-11681: --- brumi1024 commented on PR #6834: URL: https://github.com/apache/hadoop/pull/6834#issuecomment-2122755033 Thanks @K0K0V0K and @p-szucs, fixed, let's wait for a build. > Update the cgroup documentation with v2 support > --- > > Key: YARN-11681 > URL: https://issues.apache.org/jira/browse/YARN-11681 > Project: Hadoop YARN > Issue Type: Sub-task >Reporter: Benjamin Teke >Assignee: Benjamin Teke >Priority: Major > Labels: pull-request-available > > Update the related > [documentation|https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/NodeManagerCgroups.html] > with v2 support. -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11681) Update the cgroup documentation with v2 support
[ https://issues.apache.org/jira/browse/YARN-11681?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17848224#comment-17848224 ] ASF GitHub Bot commented on YARN-11681: --- p-szucs commented on PR #6834: URL: https://github.com/apache/hadoop/pull/6834#issuecomment-2122735604 Thanks @brumi1024, agree with @K0K0V0K, LGTM too, I can only see the blanks error on the yetus results. > Update the cgroup documentation with v2 support > --- > > Key: YARN-11681 > URL: https://issues.apache.org/jira/browse/YARN-11681 > Project: Hadoop YARN > Issue Type: Sub-task >Reporter: Benjamin Teke >Assignee: Benjamin Teke >Priority: Major > Labels: pull-request-available > > Update the related > [documentation|https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/NodeManagerCgroups.html] > with v2 support. -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11681) Update the cgroup documentation with v2 support
[
https://issues.apache.org/jira/browse/YARN-11681?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17848148#comment-17848148
]
ASF GitHub Bot commented on YARN-11681:
---
hadoop-yetus commented on PR #6834:
URL: https://github.com/apache/hadoop/pull/6834#issuecomment-2122258281
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 0m 19s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 0s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available.
|
| +0 :ok: | markdownlint | 0m 0s | | markdownlint was not available.
|
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
_ trunk Compile Tests _ |
| +1 :green_heart: | mvninstall | 32m 19s | | trunk passed |
| +1 :green_heart: | mvnsite | 0m 17s | | trunk passed |
| +1 :green_heart: | shadedclient | 51m 36s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +1 :green_heart: | mvninstall | 0m 10s | | the patch passed |
| -1 :x: | blanks | 0m 0s |
[/blanks-eol.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6834/2/artifact/out/blanks-eol.txt)
| The patch has 1 line(s) that end in blanks. Use git apply --whitespace=fix
<>. Refer https://git-scm.com/docs/git-apply |
| +1 :green_heart: | mvnsite | 0m 12s | | the patch passed |
| +1 :green_heart: | shadedclient | 19m 14s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart: | asflicense | 0m 24s | | The patch does not
generate ASF License warnings. |
| | | 75m 17s | | |
| Subsystem | Report/Notes |
|--:|:-|
| Docker | ClientAPI=1.44 ServerAPI=1.44 base:
https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6834/2/artifact/out/Dockerfile
|
| GITHUB PR | https://github.com/apache/hadoop/pull/6834 |
| Optional Tests | dupname asflicense mvnsite codespell detsecrets
markdownlint |
| uname | Linux 36006464e442 5.15.0-94-generic #104-Ubuntu SMP Tue Jan 9
15:25:40 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / 539d6f45cbf617ac204e98a062d5d2e8396efa4e |
| Max. process+thread count | 551 (vs. ulimit of 5500) |
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site U:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site |
| Console output |
https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6834/2/console |
| versions | git=2.25.1 maven=3.6.3 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
> Update the cgroup documentation with v2 support
> ---
>
> Key: YARN-11681
> URL: https://issues.apache.org/jira/browse/YARN-11681
> Project: Hadoop YARN
> Issue Type: Sub-task
>Reporter: Benjamin Teke
>Assignee: Benjamin Teke
>Priority: Major
> Labels: pull-request-available
>
> Update the related
> [documentation|https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/NodeManagerCgroups.html]
> with v2 support.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11697) Fix fair scheduler race condition in removeApplicationAttempt and moveApplication
[
https://issues.apache.org/jira/browse/YARN-11697?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17848123#comment-17848123
]
Wilfred Spiegelenburg commented on YARN-11697:
--
You need to figure out why you get two remove events in a row for the same
application. This code has not change in multiple years. If this was really a
big issue we should have seen this happen more often and years ago.
Try to reproduce without the backports and see if it still happens. You might
have backported things that are not compatible that cause side effects.
> Fix fair scheduler race condition in removeApplicationAttempt and
> moveApplication
> -
>
> Key: YARN-11697
> URL: https://issues.apache.org/jira/browse/YARN-11697
> Project: Hadoop YARN
> Issue Type: Bug
>Affects Versions: 3.2.1
>Reporter: Syed Shameerur Rahman
>Assignee: Syed Shameerur Rahman
>Priority: Major
>
> For Hadoop version 3.2.1, the ResourceManager (RM) restarts frequently with
> the following exception
> {code:java}
> 2024-03-11 04:41:29,329 FATAL org.apache.hadoop.yarn.event.EventDispatcher
> (SchedulerEventDispatcher:Event Processor): Error in handling event type
> APP_ATTEMPT_REMOVED to the Event Dispatcher
> java.lang.IllegalStateException: Given app to remove
> appattempt_1706879498319_86660_01 Alloc: does not
> exist in queue [root, demand=,
> running=, share=, w=1.0]
> at
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue.removeApp(FSLeafQueue.java:121)
> at
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.removeApplicationAttempt(FairScheduler.java:757)
> at
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.handle(FairScheduler.java:1378)
> at
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.handle(FairScheduler.java:139)
> at
> org.apache.hadoop.yarn.event.EventDispatcher$EventProcessor.run(EventDispatcher.java:66)
> at java.lang.Thread.run(Thread.java:750)
> {code}
> The exception seems similar to the one mentioned in YARN-5136, but it looks
> like there is still some edge cases not covered by YARN-5136.
> 1. On deeper look, i could see that as mentioned in the comment here. if a
> call for a moveApplication and removeApplicationAttempt for the same attempt
> are processed in short succession the application attempt will still contain
> a queue reference but is already removed from the list of applications for
> the queue.
> 2. This can happen when
> [moveApplication|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L1908]
> removes the appAttempt from the queue and
> [removeApplicationAttempt|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L707]
> also tries to remove the same appAttempt from the queue.
> 3. On further checking, i could see that before doing
> [moveApplication|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L1779]
> writeLock on appAttempt is taken where as for
> [removeApplicationAttempt|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L665]
> , i don't see any writelock being taken which can result in race condition
> if same appAttempt is being processed.
> 4. Additionally as mentioned in the comment here when such scenario occurs
> ideally we should not take down RM.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11697) Fix fair scheduler race condition in removeApplicationAttempt and moveApplication
[
https://issues.apache.org/jira/browse/YARN-11697?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17848119#comment-17848119
]
Syed Shameerur Rahman commented on YARN-11697:
--
[~wilfreds]
# IMHO, when the appAttempt is not available in the queue to be removed, It
should be handled more gracefully than throwing IllegalStateException which
will take down the RM.
# Since the appAttempt is anyhow not available in the queue we can safely
throw warning message instead of throwing exception
Any thoughts on the above approach ?
> Fix fair scheduler race condition in removeApplicationAttempt and
> moveApplication
> -
>
> Key: YARN-11697
> URL: https://issues.apache.org/jira/browse/YARN-11697
> Project: Hadoop YARN
> Issue Type: Bug
>Affects Versions: 3.2.1
>Reporter: Syed Shameerur Rahman
>Assignee: Syed Shameerur Rahman
>Priority: Major
>
> For Hadoop version 3.2.1, the ResourceManager (RM) restarts frequently with
> the following exception
> {code:java}
> 2024-03-11 04:41:29,329 FATAL org.apache.hadoop.yarn.event.EventDispatcher
> (SchedulerEventDispatcher:Event Processor): Error in handling event type
> APP_ATTEMPT_REMOVED to the Event Dispatcher
> java.lang.IllegalStateException: Given app to remove
> appattempt_1706879498319_86660_01 Alloc: does not
> exist in queue [root, demand=,
> running=, share=, w=1.0]
> at
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue.removeApp(FSLeafQueue.java:121)
> at
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.removeApplicationAttempt(FairScheduler.java:757)
> at
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.handle(FairScheduler.java:1378)
> at
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.handle(FairScheduler.java:139)
> at
> org.apache.hadoop.yarn.event.EventDispatcher$EventProcessor.run(EventDispatcher.java:66)
> at java.lang.Thread.run(Thread.java:750)
> {code}
> The exception seems similar to the one mentioned in YARN-5136, but it looks
> like there is still some edge cases not covered by YARN-5136.
> 1. On deeper look, i could see that as mentioned in the comment here. if a
> call for a moveApplication and removeApplicationAttempt for the same attempt
> are processed in short succession the application attempt will still contain
> a queue reference but is already removed from the list of applications for
> the queue.
> 2. This can happen when
> [moveApplication|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L1908]
> removes the appAttempt from the queue and
> [removeApplicationAttempt|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L707]
> also tries to remove the same appAttempt from the queue.
> 3. On further checking, i could see that before doing
> [moveApplication|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L1779]
> writeLock on appAttempt is taken where as for
> [removeApplicationAttempt|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L665]
> , i don't see any writelock being taken which can result in race condition
> if same appAttempt is being processed.
> 4. Additionally as mentioned in the comment here when such scenario occurs
> ideally we should not take down RM.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Comment Edited] (YARN-11697) Fix fair scheduler race condition in removeApplicationAttempt and moveApplication
[
https://issues.apache.org/jira/browse/YARN-11697?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17848086#comment-17848086
]
Syed Shameerur Rahman edited comment on YARN-11697 at 5/21/24 8:00 AM:
---
Additionally i could specifically see this when Application is being killed and
corresponds to the following code
{code:java}
boolean removeApp(FSAppAttempt app) {
boolean runnable = false;
// Remove app from runnable/nonRunnable list while holding the write lock
writeLock.lock();
try {
runnable = runnableApps.remove(app);
if (!runnable) {
// removeNonRunnableApp acquires the write lock again, which is fine
if (!removeNonRunnableApp(app)) {
throw new IllegalStateException("Given app to remove " + app +
" does not exist in queue " + this);
}
}
} finally {
writeLock.unlock();
} {code}
was (Author: srahman):
Additionally i could specifically see this when Application is being killed.
> Fix fair scheduler race condition in removeApplicationAttempt and
> moveApplication
> -
>
> Key: YARN-11697
> URL: https://issues.apache.org/jira/browse/YARN-11697
> Project: Hadoop YARN
> Issue Type: Bug
>Affects Versions: 3.2.1
>Reporter: Syed Shameerur Rahman
>Assignee: Syed Shameerur Rahman
>Priority: Major
>
> For Hadoop version 3.2.1, the ResourceManager (RM) restarts frequently with
> the following exception
> {code:java}
> 2024-03-11 04:41:29,329 FATAL org.apache.hadoop.yarn.event.EventDispatcher
> (SchedulerEventDispatcher:Event Processor): Error in handling event type
> APP_ATTEMPT_REMOVED to the Event Dispatcher
> java.lang.IllegalStateException: Given app to remove
> appattempt_1706879498319_86660_01 Alloc: does not
> exist in queue [root, demand=,
> running=, share=, w=1.0]
> at
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue.removeApp(FSLeafQueue.java:121)
> at
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.removeApplicationAttempt(FairScheduler.java:757)
> at
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.handle(FairScheduler.java:1378)
> at
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.handle(FairScheduler.java:139)
> at
> org.apache.hadoop.yarn.event.EventDispatcher$EventProcessor.run(EventDispatcher.java:66)
> at java.lang.Thread.run(Thread.java:750)
> {code}
> The exception seems similar to the one mentioned in YARN-5136, but it looks
> like there is still some edge cases not covered by YARN-5136.
> 1. On deeper look, i could see that as mentioned in the comment here. if a
> call for a moveApplication and removeApplicationAttempt for the same attempt
> are processed in short succession the application attempt will still contain
> a queue reference but is already removed from the list of applications for
> the queue.
> 2. This can happen when
> [moveApplication|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L1908]
> removes the appAttempt from the queue and
> [removeApplicationAttempt|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L707]
> also tries to remove the same appAttempt from the queue.
> 3. On further checking, i could see that before doing
> [moveApplication|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L1779]
> writeLock on appAttempt is taken where as for
> [removeApplicationAttempt|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L665]
> , i don't see any writelock being taken which can result in race condition
> if same appAttempt is being processed.
> 4. Additionally as mentioned in the comment here when such scenario occurs
> ideally we should not take down RM.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11697) Fix fair scheduler race condition in removeApplicationAttempt and moveApplication
[
https://issues.apache.org/jira/browse/YARN-11697?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17848086#comment-17848086
]
Syed Shameerur Rahman commented on YARN-11697:
--
Additionally i could specifically see this when Application is being killed.
> Fix fair scheduler race condition in removeApplicationAttempt and
> moveApplication
> -
>
> Key: YARN-11697
> URL: https://issues.apache.org/jira/browse/YARN-11697
> Project: Hadoop YARN
> Issue Type: Bug
>Affects Versions: 3.2.1
>Reporter: Syed Shameerur Rahman
>Assignee: Syed Shameerur Rahman
>Priority: Major
>
> For Hadoop version 3.2.1, the ResourceManager (RM) restarts frequently with
> the following exception
> {code:java}
> 2024-03-11 04:41:29,329 FATAL org.apache.hadoop.yarn.event.EventDispatcher
> (SchedulerEventDispatcher:Event Processor): Error in handling event type
> APP_ATTEMPT_REMOVED to the Event Dispatcher
> java.lang.IllegalStateException: Given app to remove
> appattempt_1706879498319_86660_01 Alloc: does not
> exist in queue [root, demand=,
> running=, share=, w=1.0]
> at
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue.removeApp(FSLeafQueue.java:121)
> at
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.removeApplicationAttempt(FairScheduler.java:757)
> at
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.handle(FairScheduler.java:1378)
> at
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.handle(FairScheduler.java:139)
> at
> org.apache.hadoop.yarn.event.EventDispatcher$EventProcessor.run(EventDispatcher.java:66)
> at java.lang.Thread.run(Thread.java:750)
> {code}
> The exception seems similar to the one mentioned in YARN-5136, but it looks
> like there is still some edge cases not covered by YARN-5136.
> 1. On deeper look, i could see that as mentioned in the comment here. if a
> call for a moveApplication and removeApplicationAttempt for the same attempt
> are processed in short succession the application attempt will still contain
> a queue reference but is already removed from the list of applications for
> the queue.
> 2. This can happen when
> [moveApplication|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L1908]
> removes the appAttempt from the queue and
> [removeApplicationAttempt|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L707]
> also tries to remove the same appAttempt from the queue.
> 3. On further checking, i could see that before doing
> [moveApplication|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L1779]
> writeLock on appAttempt is taken where as for
> [removeApplicationAttempt|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L665]
> , i don't see any writelock being taken which can result in race condition
> if same appAttempt is being processed.
> 4. Additionally as mentioned in the comment here when such scenario occurs
> ideally we should not take down RM.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11697) Fix fair scheduler race condition in removeApplicationAttempt and moveApplication
[
https://issues.apache.org/jira/browse/YARN-11697?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17848085#comment-17848085
]
Syed Shameerur Rahman commented on YARN-11697:
--
[~wilfreds]
I had some custom code/backports from higher version and hence the code lines
might have differed from the OSS hadoop code base. I could see the following
exception though
java.lang.IllegalStateException: Given app to remove
appattempt_1706879498319_86660_01 Alloc: does not
exist in queue [root, demand=,
running=, share=, w=1.0]
So this exception comes only when the appAttempt is already removed from the
queue and we try to remove it again. Throwing IllegalStateException causes the
RM to shutdown with exception. Can you think of any scenario this can happen ?
> Fix fair scheduler race condition in removeApplicationAttempt and
> moveApplication
> -
>
> Key: YARN-11697
> URL: https://issues.apache.org/jira/browse/YARN-11697
> Project: Hadoop YARN
> Issue Type: Bug
>Affects Versions: 3.2.1
>Reporter: Syed Shameerur Rahman
>Assignee: Syed Shameerur Rahman
>Priority: Major
>
> For Hadoop version 3.2.1, the ResourceManager (RM) restarts frequently with
> the following exception
> {code:java}
> 2024-03-11 04:41:29,329 FATAL org.apache.hadoop.yarn.event.EventDispatcher
> (SchedulerEventDispatcher:Event Processor): Error in handling event type
> APP_ATTEMPT_REMOVED to the Event Dispatcher
> java.lang.IllegalStateException: Given app to remove
> appattempt_1706879498319_86660_01 Alloc: does not
> exist in queue [root, demand=,
> running=, share=, w=1.0]
> at
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue.removeApp(FSLeafQueue.java:121)
> at
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.removeApplicationAttempt(FairScheduler.java:757)
> at
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.handle(FairScheduler.java:1378)
> at
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.handle(FairScheduler.java:139)
> at
> org.apache.hadoop.yarn.event.EventDispatcher$EventProcessor.run(EventDispatcher.java:66)
> at java.lang.Thread.run(Thread.java:750)
> {code}
> The exception seems similar to the one mentioned in YARN-5136, but it looks
> like there is still some edge cases not covered by YARN-5136.
> 1. On deeper look, i could see that as mentioned in the comment here. if a
> call for a moveApplication and removeApplicationAttempt for the same attempt
> are processed in short succession the application attempt will still contain
> a queue reference but is already removed from the list of applications for
> the queue.
> 2. This can happen when
> [moveApplication|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L1908]
> removes the appAttempt from the queue and
> [removeApplicationAttempt|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L707]
> also tries to remove the same appAttempt from the queue.
> 3. On further checking, i could see that before doing
> [moveApplication|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L1779]
> writeLock on appAttempt is taken where as for
> [removeApplicationAttempt|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L665]
> , i don't see any writelock being taken which can result in race condition
> if same appAttempt is being processed.
> 4. Additionally as mentioned in the comment here when such scenario occurs
> ideally we should not take down RM.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11697) Fix fair scheduler race condition in removeApplicationAttempt and moveApplication
[
https://issues.apache.org/jira/browse/YARN-11697?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17848072#comment-17848072
]
Wilfred Spiegelenburg commented on YARN-11697:
--
The stack trace does not correspond to hadoop 3.2.1:
[FairScheduler.java:757|https://github.com/apache/hadoop/blob/branch-3.2.1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L757]
That points to this line in hadoop 3.2.1 which is part of
completedContainerInternal
{code:java}
755 application.containerCompleted(rmContainer, containerStatus, event);
756 if (node != null) {
757 node.releaseContainer(rmContainer.getContainerId(), false);
758 } else if (LOG.isDebugEnabled()) {
759 LOG.debug("Skipping container release on removed node: " + nodeID);
760 } {code}
The comment in the moveApplication around locking the app attempt are for
scheduling. An application could be scheduled while being moved and that needs
to be stopped. The remove of an application attempt takes a write lock on the
scheduler itself. Same as the move does. So a moveApplication and
removeApplicationAttempt cannot happen at the same time. they both need that
lock and are serialised.
I think you are looking at the wrong thing and a move is not involved.
> Fix fair scheduler race condition in removeApplicationAttempt and
> moveApplication
> -
>
> Key: YARN-11697
> URL: https://issues.apache.org/jira/browse/YARN-11697
> Project: Hadoop YARN
> Issue Type: Bug
>Affects Versions: 3.2.1
>Reporter: Syed Shameerur Rahman
>Assignee: Syed Shameerur Rahman
>Priority: Major
>
> For Hadoop version 3.2.1, the ResourceManager (RM) restarts frequently with
> the following exception
> {code:java}
> 2024-03-11 04:41:29,329 FATAL org.apache.hadoop.yarn.event.EventDispatcher
> (SchedulerEventDispatcher:Event Processor): Error in handling event type
> APP_ATTEMPT_REMOVED to the Event Dispatcher
> java.lang.IllegalStateException: Given app to remove
> appattempt_1706879498319_86660_01 Alloc: does not
> exist in queue [root, demand=,
> running=, share=, w=1.0]
> at
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue.removeApp(FSLeafQueue.java:121)
> at
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.removeApplicationAttempt(FairScheduler.java:757)
> at
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.handle(FairScheduler.java:1378)
> at
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.handle(FairScheduler.java:139)
> at
> org.apache.hadoop.yarn.event.EventDispatcher$EventProcessor.run(EventDispatcher.java:66)
> at java.lang.Thread.run(Thread.java:750)
> {code}
> The exception seems similar to the one mentioned in YARN-5136, but it looks
> like there is still some edge cases not covered by YARN-5136.
> 1. On deeper look, i could see that as mentioned in the comment here. if a
> call for a moveApplication and removeApplicationAttempt for the same attempt
> are processed in short succession the application attempt will still contain
> a queue reference but is already removed from the list of applications for
> the queue.
> 2. This can happen when
> [moveApplication|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L1908]
> removes the appAttempt from the queue and
> [removeApplicationAttempt|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L707]
> also tries to remove the same appAttempt from the queue.
> 3. On further checking, i could see that before doing
> [moveApplication|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L1779]
> writeLock on appAttempt is taken where as for
> [removeApplicationAttempt|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair
[jira] [Updated] (YARN-11697) Fix fair scheduler race condition in removeApplicationAttempt and moveApplication
[
https://issues.apache.org/jira/browse/YARN-11697?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Syed Shameerur Rahman updated YARN-11697:
-
Description:
For Hadoop version 3.2.1, the ResourceManager (RM) restarts frequently with the
following exception
{code:java}
2024-03-11 04:41:29,329 FATAL org.apache.hadoop.yarn.event.EventDispatcher
(SchedulerEventDispatcher:Event Processor): Error in handling event type
APP_ATTEMPT_REMOVED to the Event Dispatcher
java.lang.IllegalStateException: Given app to remove
appattempt_1706879498319_86660_01 Alloc: does not
exist in queue [root, demand=,
running=, share=, w=1.0]
at
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue.removeApp(FSLeafQueue.java:121)
at
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.removeApplicationAttempt(FairScheduler.java:757)
at
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.handle(FairScheduler.java:1378)
at
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.handle(FairScheduler.java:139)
at
org.apache.hadoop.yarn.event.EventDispatcher$EventProcessor.run(EventDispatcher.java:66)
at java.lang.Thread.run(Thread.java:750)
{code}
The exception seems similar to the one mentioned in YARN-5136, but it looks
like there is still some edge cases not covered by YARN-5136.
1. On deeper look, i could see that as mentioned in the comment here. if a call
for a moveApplication and removeApplicationAttempt for the same attempt are
processed in short succession the application attempt will still contain a
queue reference but is already removed from the list of applications for the
queue.
2. This can happen when
[moveApplication|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L1908]
removes the appAttempt from the queue and
[removeApplicationAttempt|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L707]
also tries to remove the same appAttempt from the queue.
3. On further checking, i could see that before doing
[moveApplication|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L1779]
writeLock on appAttempt is taken where as for
[removeApplicationAttempt|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L665]
, i don't see any writelock being taken which can result in race condition if
same appAttempt is being processed.
4. Additionally as mentioned in the comment here when such scenario occurs
ideally we should not take down RM.
was:
For Hadoop version 3.2.1, the ResourceManager (RM) restarts frequently with the
following exception
{code:java}
2024-03-11 04:41:29,329 FATAL org.apache.hadoop.yarn.event.EventDispatcher
(SchedulerEventDispatcher:Event Processor): Error in handling event type
APP_ATTEMPT_REMOVED to the Event Dispatcher
java.lang.IllegalStateException: Given app to remove
appattempt_1706879498319_86660_01 Alloc: does not
exist in queue [root.tier2.livy, demand=,
running=, share=, w=1.0]
at
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue.removeApp(FSLeafQueue.java:121)
at
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.removeApplicationAttempt(FairScheduler.java:757)
at
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.handle(FairScheduler.java:1378)
at
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.handle(FairScheduler.java:139)
at
org.apache.hadoop.yarn.event.EventDispatcher$EventProcessor.run(EventDispatcher.java:66)
at java.lang.Thread.run(Thread.java:750)
{code}
The exception seems similar to the one mentioned in YARN-5136, but it looks
like there is still some edge cases not covered by YARN-5136.
1. On deeper look, i could see that as mentioned in the comment here. if a call
for a moveApplication and removeApplicationAttempt for the same attempt are
processed in short succession the application attempt will still contain a
queue reference but is already removed from the list of applications for the
queue.
2. This can happen when
[moveApplication|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn
[jira] [Created] (YARN-11697) Fix fair scheduler race condition in removeApplicationAttempt and moveApplication
Syed Shameerur Rahman created YARN-11697:
Summary: Fix fair scheduler race condition in
removeApplicationAttempt and moveApplication
Key: YARN-11697
URL: https://issues.apache.org/jira/browse/YARN-11697
Project: Hadoop YARN
Issue Type: Bug
Affects Versions: 3.2.1
Reporter: Syed Shameerur Rahman
Assignee: Syed Shameerur Rahman
For Hadoop version 3.2.1, the ResourceManager (RM) restarts frequently with the
following exception
{code:java}
2024-03-11 04:41:29,329 FATAL org.apache.hadoop.yarn.event.EventDispatcher
(SchedulerEventDispatcher:Event Processor): Error in handling event type
APP_ATTEMPT_REMOVED to the Event Dispatcher
java.lang.IllegalStateException: Given app to remove
appattempt_1706879498319_86660_01 Alloc: does not
exist in queue [root.tier2.livy, demand=,
running=, share=, w=1.0]
at
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue.removeApp(FSLeafQueue.java:121)
at
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.removeApplicationAttempt(FairScheduler.java:757)
at
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.handle(FairScheduler.java:1378)
at
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.handle(FairScheduler.java:139)
at
org.apache.hadoop.yarn.event.EventDispatcher$EventProcessor.run(EventDispatcher.java:66)
at java.lang.Thread.run(Thread.java:750)
{code}
The exception seems similar to the one mentioned in YARN-5136, but it looks
like there is still some edge cases not covered by YARN-5136.
1. On deeper look, i could see that as mentioned in the comment here. if a call
for a moveApplication and removeApplicationAttempt for the same attempt are
processed in short succession the application attempt will still contain a
queue reference but is already removed from the list of applications for the
queue.
2. This can happen when
[moveApplication|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L1908]
removes the appAttempt from the queue and
[removeApplicationAttempt|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L707]
also tries to remove the same appAttempt from the queue.
3. On further checking, i could see that before doing
[moveApplication|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L1779]
writeLock on appAttempt is taken where as for
[removeApplicationAttempt|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L665]
, i don't see any writelock being taken which can result in race condition if
same appAttempt is being processed.
4. Additionally as mentioned in the comment here when such scenario occurs
ideally we should not take down RM.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11697) Fix fair scheduler race condition in removeApplicationAttempt and moveApplication
[
https://issues.apache.org/jira/browse/YARN-11697?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17848046#comment-17848046
]
Syed Shameerur Rahman commented on YARN-11697:
--
[~wilfreds] any thoughts on this ?
> Fix fair scheduler race condition in removeApplicationAttempt and
> moveApplication
> -
>
> Key: YARN-11697
> URL: https://issues.apache.org/jira/browse/YARN-11697
> Project: Hadoop YARN
> Issue Type: Bug
>Affects Versions: 3.2.1
>Reporter: Syed Shameerur Rahman
>Assignee: Syed Shameerur Rahman
>Priority: Major
>
> For Hadoop version 3.2.1, the ResourceManager (RM) restarts frequently with
> the following exception
> {code:java}
> 2024-03-11 04:41:29,329 FATAL org.apache.hadoop.yarn.event.EventDispatcher
> (SchedulerEventDispatcher:Event Processor): Error in handling event type
> APP_ATTEMPT_REMOVED to the Event Dispatcher
> java.lang.IllegalStateException: Given app to remove
> appattempt_1706879498319_86660_01 Alloc: does not
> exist in queue [root.tier2.livy, demand=,
> running=, share=, w=1.0]
> at
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue.removeApp(FSLeafQueue.java:121)
> at
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.removeApplicationAttempt(FairScheduler.java:757)
> at
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.handle(FairScheduler.java:1378)
> at
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.handle(FairScheduler.java:139)
> at
> org.apache.hadoop.yarn.event.EventDispatcher$EventProcessor.run(EventDispatcher.java:66)
> at java.lang.Thread.run(Thread.java:750)
> {code}
> The exception seems similar to the one mentioned in YARN-5136, but it looks
> like there is still some edge cases not covered by YARN-5136.
> 1. On deeper look, i could see that as mentioned in the comment here. if a
> call for a moveApplication and removeApplicationAttempt for the same attempt
> are processed in short succession the application attempt will still contain
> a queue reference but is already removed from the list of applications for
> the queue.
> 2. This can happen when
> [moveApplication|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L1908]
> removes the appAttempt from the queue and
> [removeApplicationAttempt|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L707]
> also tries to remove the same appAttempt from the queue.
> 3. On further checking, i could see that before doing
> [moveApplication|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L1779]
> writeLock on appAttempt is taken where as for
> [removeApplicationAttempt|https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java#L665]
> , i don't see any writelock being taken which can result in race condition
> if same appAttempt is being processed.
> 4. Additionally as mentioned in the comment here when such scenario occurs
> ideally we should not take down RM.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11687) Update CGroupsResourceCalculator to track usages using cgroupv2
[
https://issues.apache.org/jira/browse/YARN-11687?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17847443#comment-17847443
]
ASF GitHub Bot commented on YARN-11687:
---
hadoop-yetus commented on PR #6835:
URL: https://github.com/apache/hadoop/pull/6835#issuecomment-2118346701
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 18m 48s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 1s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 1s | | detect-secrets was not available.
|
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to
include 3 new or modified test files. |
_ trunk Compile Tests _ |
| +0 :ok: | mvndep | 14m 32s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 37m 38s | | trunk passed |
| +1 :green_heart: | compile | 8m 16s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | compile | 7m 18s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | checkstyle | 2m 0s | | trunk passed |
| +1 :green_heart: | mvnsite | 1m 49s | | trunk passed |
| +1 :green_heart: | javadoc | 1m 57s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javadoc | 1m 50s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | spotbugs | 3m 24s | | trunk passed |
| +1 :green_heart: | shadedclient | 39m 52s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 32s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 1m 7s | | the patch passed |
| +1 :green_heart: | compile | 7m 29s | | the patch passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javac | 7m 29s | | the patch passed |
| +1 :green_heart: | compile | 7m 16s | | the patch passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | javac | 7m 16s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks
issues. |
| -0 :warning: | checkstyle | 1m 51s |
[/results-checkstyle-hadoop-yarn-project_hadoop-yarn.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/1/artifact/out/results-checkstyle-hadoop-yarn-project_hadoop-yarn.txt)
| hadoop-yarn-project/hadoop-yarn: The patch generated 34 new + 11 unchanged
- 0 fixed = 45 total (was 11) |
| +1 :green_heart: | mvnsite | 1m 37s | | the patch passed |
| -1 :x: | javadoc | 0m 47s |
[/results-javadoc-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/1/artifact/out/results-javadoc-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt)
|
hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1
with JDK Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 generated 1 new + 98
unchanged - 1 fixed = 99 total (was 99) |
| +1 :green_heart: | javadoc | 0m 55s | | hadoop-yarn-common in the
patch passed with JDK Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06. |
| +1 :green_heart: | javadoc | 0m 44s | |
hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager-jdkPrivateBuild-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06
with JDK Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 generated 0 new +
195 unchanged - 1 fixed = 195 total (was 196) |
| -1 :x: | spotbugs | 1m 41s |
[/new-spotbugs-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager.html](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6835/1/artifact/out/new-spotbugs-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager.html)
|
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager
generated 2 new + 0 unchanged - 0 fixed = 2 total (was 0) |
| +1 :green_heart: | shadedclient | 41m 9s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart
[jira] [Commented] (YARN-11687) Update CGroupsResourceCalculator to track usages using cgroupv2
[ https://issues.apache.org/jira/browse/YARN-11687?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17847371#comment-17847371 ] ASF GitHub Bot commented on YARN-11687: --- K0K0V0K closed pull request #6780: YARN-11687. Update CGroupsResourceCalculator to track usages using cgroupv2 URL: https://github.com/apache/hadoop/pull/6780 > Update CGroupsResourceCalculator to track usages using cgroupv2 > --- > > Key: YARN-11687 > URL: https://issues.apache.org/jira/browse/YARN-11687 > Project: Hadoop YARN > Issue Type: Sub-task >Reporter: Benjamin Teke >Assignee: Bence Kosztolnik >Priority: Major > Labels: pull-request-available > > [CGroupsResourceCalculator|https://github.com/apache/hadoop/blob/f609460bda0c2bd87dd3580158e549e2f34f14d5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java] > should also be updated to handle the cgroup v2 changes. -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11687) Update CGroupsResourceCalculator to track usages using cgroupv2
[ https://issues.apache.org/jira/browse/YARN-11687?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17847370#comment-17847370 ] ASF GitHub Bot commented on YARN-11687: --- K0K0V0K opened a new pull request, #6835: URL: https://github.com/apache/hadoop/pull/6835 ### Description of PR - Create a new CGroupsV2ResourceCalculator to handle resources - Refactor existing CGroupsResourceCalculator - Move the common code to the AbstractCGroupsResourceCalculator - FIX [YARN-8037](https://issues.apache.org/jira/browse/YARN-8037): reduce waring logs - Modify CombinedResourceCalculator to first check cgroupv2 then v1 than ProcfsBasedProcessTree as source of truce ### How was this patch tested? - Deployed to an EC cluster (RHEL 8) where cgroup v1 and v2 were enabled > Update CGroupsResourceCalculator to track usages using cgroupv2 > --- > > Key: YARN-11687 > URL: https://issues.apache.org/jira/browse/YARN-11687 > Project: Hadoop YARN > Issue Type: Sub-task >Reporter: Benjamin Teke >Assignee: Bence Kosztolnik >Priority: Major > Labels: pull-request-available > > [CGroupsResourceCalculator|https://github.com/apache/hadoop/blob/f609460bda0c2bd87dd3580158e549e2f34f14d5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java] > should also be updated to handle the cgroup v2 changes. -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11681) Update the cgroup documentation with v2 support
[
https://issues.apache.org/jira/browse/YARN-11681?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17847358#comment-17847358
]
ASF GitHub Bot commented on YARN-11681:
---
hadoop-yetus commented on PR #6834:
URL: https://github.com/apache/hadoop/pull/6834#issuecomment-2117885452
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 0m 20s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 0s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available.
|
| +0 :ok: | markdownlint | 0m 0s | | markdownlint was not available.
|
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
_ trunk Compile Tests _ |
| +1 :green_heart: | mvninstall | 32m 17s | | trunk passed |
| +1 :green_heart: | mvnsite | 0m 20s | | trunk passed |
| +1 :green_heart: | shadedclient | 51m 51s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +1 :green_heart: | mvninstall | 0m 9s | | the patch passed |
| -1 :x: | blanks | 0m 0s |
[/blanks-eol.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6834/1/artifact/out/blanks-eol.txt)
| The patch has 1 line(s) that end in blanks. Use git apply --whitespace=fix
<>. Refer https://git-scm.com/docs/git-apply |
| +1 :green_heart: | mvnsite | 0m 13s | | the patch passed |
| +1 :green_heart: | shadedclient | 19m 12s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart: | asflicense | 0m 24s | | The patch does not
generate ASF License warnings. |
| | | 74m 56s | | |
| Subsystem | Report/Notes |
|--:|:-|
| Docker | ClientAPI=1.44 ServerAPI=1.44 base:
https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6834/1/artifact/out/Dockerfile
|
| GITHUB PR | https://github.com/apache/hadoop/pull/6834 |
| Optional Tests | dupname asflicense mvnsite codespell detsecrets
markdownlint |
| uname | Linux c94c08ed2f3d 5.15.0-94-generic #104-Ubuntu SMP Tue Jan 9
15:25:40 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / e58064b863f2953019a77a2b64fcbe3ab9e2bc00 |
| Max. process+thread count | 553 (vs. ulimit of 5500) |
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site U:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site |
| Console output |
https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6834/1/console |
| versions | git=2.25.1 maven=3.6.3 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
> Update the cgroup documentation with v2 support
> ---
>
> Key: YARN-11681
> URL: https://issues.apache.org/jira/browse/YARN-11681
> Project: Hadoop YARN
> Issue Type: Sub-task
>Reporter: Benjamin Teke
>Assignee: Benjamin Teke
>Priority: Major
> Labels: pull-request-available
>
> Update the related
> [documentation|https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/NodeManagerCgroups.html]
> with v2 support.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11681) Update the cgroup documentation with v2 support
[ https://issues.apache.org/jira/browse/YARN-11681?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17847343#comment-17847343 ] ASF GitHub Bot commented on YARN-11681: --- K0K0V0K commented on code in PR #6834: URL: https://github.com/apache/hadoop/pull/6834#discussion_r1605127306 ## hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerCgroups.md: ## @@ -12,51 +12,53 @@ limitations under the License. See accompanying LICENSE file. --> -Using CGroups with YARN +Using Cgroups with YARN === Update the cgroup documentation with v2 support > --- > > Key: YARN-11681 > URL: https://issues.apache.org/jira/browse/YARN-11681 > Project: Hadoop YARN > Issue Type: Sub-task >Reporter: Benjamin Teke >Assignee: Benjamin Teke >Priority: Major > Labels: pull-request-available > > Update the related > [documentation|https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/NodeManagerCgroups.html] > with v2 support. -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Assigned] (YARN-11681) Update the cgroup documentation with v2 support
[ https://issues.apache.org/jira/browse/YARN-11681?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Benjamin Teke reassigned YARN-11681: Assignee: Benjamin Teke > Update the cgroup documentation with v2 support > --- > > Key: YARN-11681 > URL: https://issues.apache.org/jira/browse/YARN-11681 > Project: Hadoop YARN > Issue Type: Sub-task >Reporter: Benjamin Teke >Assignee: Benjamin Teke >Priority: Major > Labels: pull-request-available > > Update the related > [documentation|https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/NodeManagerCgroups.html] > with v2 support. -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-11681) Update the cgroup documentation with v2 support
[ https://issues.apache.org/jira/browse/YARN-11681?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] ASF GitHub Bot updated YARN-11681: -- Labels: pull-request-available (was: ) > Update the cgroup documentation with v2 support > --- > > Key: YARN-11681 > URL: https://issues.apache.org/jira/browse/YARN-11681 > Project: Hadoop YARN > Issue Type: Sub-task >Reporter: Benjamin Teke >Priority: Major > Labels: pull-request-available > > Update the related > [documentation|https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/NodeManagerCgroups.html] > with v2 support. -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11681) Update the cgroup documentation with v2 support
[ https://issues.apache.org/jira/browse/YARN-11681?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17847336#comment-17847336 ] ASF GitHub Bot commented on YARN-11681: --- brumi1024 opened a new pull request, #6834: URL: https://github.com/apache/hadoop/pull/6834 ### Description of PR Documentation update. ### How was this patch tested? ### For code changes: - [ ] Does the title or this PR starts with the corresponding JIRA issue id (e.g. 'HADOOP-17799. Your PR title ...')? - [ ] Object storage: have the integration tests been executed and the endpoint declared according to the connector-specific documentation? - [ ] If adding new dependencies to the code, are these dependencies licensed in a way that is compatible for inclusion under [ASF 2.0](http://www.apache.org/legal/resolved.html#category-a)? - [ ] If applicable, have you updated the `LICENSE`, `LICENSE-binary`, `NOTICE-binary` files? > Update the cgroup documentation with v2 support > --- > > Key: YARN-11681 > URL: https://issues.apache.org/jira/browse/YARN-11681 > Project: Hadoop YARN > Issue Type: Sub-task >Reporter: Benjamin Teke >Priority: Major > > Update the related > [documentation|https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/NodeManagerCgroups.html] > with v2 support. -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11687) Update CGroupsResourceCalculator to track usages using cgroupv2
[ https://issues.apache.org/jira/browse/YARN-11687?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17847253#comment-17847253 ] ASF GitHub Bot commented on YARN-11687: --- hadoop-yetus commented on PR #6780: URL: https://github.com/apache/hadoop/pull/6780#issuecomment-2117460222 :broken_heart: **-1 overall** | Vote | Subsystem | Runtime | Logfile | Comment | |::|--:|:|::|:---:| | +0 :ok: | reexec | 0m 0s | | Docker mode activated. | | -1 :x: | patch | 0m 20s | | https://github.com/apache/hadoop/pull/6780 does not apply to trunk. Rebase required? Wrong Branch? See https://cwiki.apache.org/confluence/display/HADOOP/How+To+Contribute for help. | | Subsystem | Report/Notes | |--:|:-| | GITHUB PR | https://github.com/apache/hadoop/pull/6780 | | Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6780/15/console | | versions | git=2.34.1 | | Powered by | Apache Yetus 0.14.0 https://yetus.apache.org | This message was automatically generated. > Update CGroupsResourceCalculator to track usages using cgroupv2 > --- > > Key: YARN-11687 > URL: https://issues.apache.org/jira/browse/YARN-11687 > Project: Hadoop YARN > Issue Type: Sub-task >Reporter: Benjamin Teke >Assignee: Bence Kosztolnik >Priority: Major > Labels: pull-request-available > > [CGroupsResourceCalculator|https://github.com/apache/hadoop/blob/f609460bda0c2bd87dd3580158e549e2f34f14d5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java] > should also be updated to handle the cgroup v2 changes. -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11687) Update CGroupsResourceCalculator to track usages using cgroupv2
[ https://issues.apache.org/jira/browse/YARN-11687?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17847252#comment-17847252 ] ASF GitHub Bot commented on YARN-11687: --- hadoop-yetus commented on PR #6780: URL: https://github.com/apache/hadoop/pull/6780#issuecomment-2117450177 :broken_heart: **-1 overall** | Vote | Subsystem | Runtime | Logfile | Comment | |::|--:|:|::|:---:| | +0 :ok: | reexec | 0m 0s | | Docker mode activated. | | -1 :x: | patch | 0m 20s | | https://github.com/apache/hadoop/pull/6780 does not apply to trunk. Rebase required? Wrong Branch? See https://cwiki.apache.org/confluence/display/HADOOP/How+To+Contribute for help. | | Subsystem | Report/Notes | |--:|:-| | GITHUB PR | https://github.com/apache/hadoop/pull/6780 | | Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6780/14/console | | versions | git=2.34.1 | | Powered by | Apache Yetus 0.14.0 https://yetus.apache.org | This message was automatically generated. > Update CGroupsResourceCalculator to track usages using cgroupv2 > --- > > Key: YARN-11687 > URL: https://issues.apache.org/jira/browse/YARN-11687 > Project: Hadoop YARN > Issue Type: Sub-task >Reporter: Benjamin Teke >Assignee: Bence Kosztolnik >Priority: Major > Labels: pull-request-available > > [CGroupsResourceCalculator|https://github.com/apache/hadoop/blob/f609460bda0c2bd87dd3580158e549e2f34f14d5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java] > should also be updated to handle the cgroup v2 changes. -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11471) FederationStateStoreFacade Cache Support Caffeine
[
https://issues.apache.org/jira/browse/YARN-11471?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17847013#comment-17847013
]
ASF GitHub Bot commented on YARN-11471:
---
hadoop-yetus commented on PR #6795:
URL: https://github.com/apache/hadoop/pull/6795#issuecomment-2115593583
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 02s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 02s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 02s | | detect-secrets was not available.
|
| +0 :ok: | shellcheck | 0m 02s | | Shellcheck was not available. |
| +0 :ok: | shelldocs | 0m 02s | | Shelldocs was not available. |
| +0 :ok: | xmllint | 0m 01s | | xmllint was not available. |
| +0 :ok: | spotbugs | 0m 01s | | spotbugs executables are not
available. |
| +0 :ok: | markdownlint | 0m 01s | | markdownlint was not available.
|
| +1 :green_heart: | @author | 0m 00s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 00s | | The patch appears to
include 1 new or modified test files. |
_ trunk Compile Tests _ |
| +0 :ok: | mvndep | 4m 20s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 117m 00s | | trunk passed |
| +1 :green_heart: | compile | 52m 52s | | trunk passed |
| +1 :green_heart: | checkstyle | 7m 51s | | trunk passed |
| -1 :x: | mvnsite | 33m 39s |
[/branch-mvnsite-root.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6795/9/artifact/out/branch-mvnsite-root.txt)
| root in trunk failed. |
| +1 :green_heart: | javadoc | 20m 56s | | trunk passed |
| +1 :green_heart: | shadedclient | 250m 07s | | branch has no errors
when building and testing our client artifacts. |
| -0 :warning: | patch | 253m 56s | | Used diff version of patch file.
Binary files and potentially other changes not applied. Please rebase and
squash commits if necessary. |
_ Patch Compile Tests _ |
| +0 :ok: | mvndep | 3m 05s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 123m 48s | | the patch passed |
| +1 :green_heart: | compile | 52m 55s | | the patch passed |
| +1 :green_heart: | javac | 52m 55s | | the patch passed |
| +1 :green_heart: | blanks | 0m 01s | | The patch has no blanks
issues. |
| +1 :green_heart: | checkstyle | 8m 23s | | the patch passed |
| -1 :x: | mvnsite | 31m 50s |
[/patch-mvnsite-root.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6795/9/artifact/out/patch-mvnsite-root.txt)
| root in the patch failed. |
| +1 :green_heart: | javadoc | 21m 06s | | the patch passed |
| +1 :green_heart: | shadedclient | 266m 42s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart: | asflicense | 9m 50s | | The patch does not
generate ASF License warnings. |
| | | 890m 05s | | |
| Subsystem | Report/Notes |
|--:|:-|
| GITHUB PR | https://github.com/apache/hadoop/pull/6795 |
| Optional Tests | dupname asflicense codespell detsecrets shellcheck
shelldocs compile javac javadoc mvninstall mvnsite unit shadedclient xmllint
spotbugs checkstyle markdownlint |
| uname | MINGW64_NT-10.0-17763 1b8708b85dfd 3.4.10-87d57229.x86_64
2024-02-14 20:17 UTC x86_64 Msys |
| Build tool | maven |
| Personality | /c/hadoop/dev-support/bin/hadoop.sh |
| git revision | trunk / e1e3e7441d3a3b92092fe573e9beb5dd524d324a |
| Default Java | Azul Systems, Inc.-1.8.0_332-b09 |
| Test Results |
https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6795/9/testReport/
|
| modules | C: hadoop-project
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site . U: . |
| Console output |
https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6795/9/console
|
| versions | git=2.44.0.windows.1 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
> FederationStateStoreFacade Cache Support Caffeine
> -
>
> Key: YARN-11471
> URL: https://issues.apache.org/jira/browse/YARN-11471
> Project: Hadoop YARN
> Is
[jira] [Commented] (YARN-11469) Refactor FederationStateStoreFacade Cache Code
[ https://issues.apache.org/jira/browse/YARN-11469?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17846813#comment-17846813 ] ASF GitHub Bot commented on YARN-11469: --- slfan1989 commented on PR #6796: URL: https://github.com/apache/hadoop/pull/6796#issuecomment-2114051501 > I don't write/review yarn code, or hdfs. better for everyone. fewer regressions and I can stay out of support escalations. sorry Thank you for feedback! I understand and respect your principles. > Refactor FederationStateStoreFacade Cache Code > -- > > Key: YARN-11469 > URL: https://issues.apache.org/jira/browse/YARN-11469 > Project: Hadoop YARN > Issue Type: Improvement > Components: federation >Affects Versions: 3.4.0 >Reporter: Shilun Fan >Assignee: Shilun Fan >Priority: Major > Labels: pull-request-available > Fix For: 3.4.0 > > > The Cache of FederationStateStoreFacade uses JCache, but considering that > JCache is not a general Cache implementation (the latest version was released > in 2014), this part of the code is refactored to support multiple Cache in > the future. -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11471) FederationStateStoreFacade Cache Support Caffeine
[ https://issues.apache.org/jira/browse/YARN-11471?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17846768#comment-17846768 ] ASF GitHub Bot commented on YARN-11471: --- slfan1989 commented on PR #6795: URL: https://github.com/apache/hadoop/pull/6795#issuecomment-2113650049 @goiri Can you help review this pr again? Thank you very much! The error reported by mvn compilation is not caused by our changes. > FederationStateStoreFacade Cache Support Caffeine > - > > Key: YARN-11471 > URL: https://issues.apache.org/jira/browse/YARN-11471 > Project: Hadoop YARN > Issue Type: Improvement > Components: federation >Affects Versions: 3.4.0 >Reporter: Shilun Fan >Assignee: Shilun Fan >Priority: Major > Labels: pull-request-available > > FederationStateStoreFacade Cache Support Caffeine -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Resolved] (YARN-11692) Support mixed cgroup v1/v2 controller structure
[ https://issues.apache.org/jira/browse/YARN-11692?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Benjamin Teke resolved YARN-11692. -- Hadoop Flags: Reviewed Target Version/s: 3.5.0 Resolution: Fixed > Support mixed cgroup v1/v2 controller structure > --- > > Key: YARN-11692 > URL: https://issues.apache.org/jira/browse/YARN-11692 > Project: Hadoop YARN > Issue Type: Sub-task >Reporter: Benjamin Teke >Assignee: Peter Szucs >Priority: Major > Labels: pull-request-available > > There were heavy changes on the device side in cgroup v2. To keep supporting > FGPAs and GPUs short term, mixed structures where some of the cgroup > controllers are from v1 while others from v2 should be supported. More info: > https://dropbear.xyz/2023/05/23/devices-with-cgroup-v2/ -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11692) Support mixed cgroup v1/v2 controller structure
[ https://issues.apache.org/jira/browse/YARN-11692?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17846663#comment-17846663 ] ASF GitHub Bot commented on YARN-11692: --- brumi1024 merged PR #6821: URL: https://github.com/apache/hadoop/pull/6821 > Support mixed cgroup v1/v2 controller structure > --- > > Key: YARN-11692 > URL: https://issues.apache.org/jira/browse/YARN-11692 > Project: Hadoop YARN > Issue Type: Sub-task >Reporter: Benjamin Teke >Assignee: Peter Szucs >Priority: Major > Labels: pull-request-available > > There were heavy changes on the device side in cgroup v2. To keep supporting > FGPAs and GPUs short term, mixed structures where some of the cgroup > controllers are from v1 while others from v2 should be supported. More info: > https://dropbear.xyz/2023/05/23/devices-with-cgroup-v2/ -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11469) Refactor FederationStateStoreFacade Cache Code
[ https://issues.apache.org/jira/browse/YARN-11469?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17846627#comment-17846627 ] ASF GitHub Bot commented on YARN-11469: --- steveloughran commented on PR #6796: URL: https://github.com/apache/hadoop/pull/6796#issuecomment-2112517978 I don't write/review yarn code, or hdfs. better for everyone. fewer regressions and I can stay out of support escalations. sorry > Refactor FederationStateStoreFacade Cache Code > -- > > Key: YARN-11469 > URL: https://issues.apache.org/jira/browse/YARN-11469 > Project: Hadoop YARN > Issue Type: Improvement > Components: federation >Affects Versions: 3.4.0 >Reporter: Shilun Fan >Assignee: Shilun Fan >Priority: Major > Labels: pull-request-available > Fix For: 3.4.0 > > > The Cache of FederationStateStoreFacade uses JCache, but considering that > JCache is not a general Cache implementation (the latest version was released > in 2014), this part of the code is refactored to support multiple Cache in > the future. -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11471) FederationStateStoreFacade Cache Support Caffeine
[
https://issues.apache.org/jira/browse/YARN-11471?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17846600#comment-17846600
]
ASF GitHub Bot commented on YARN-11471:
---
hadoop-yetus commented on PR #6795:
URL: https://github.com/apache/hadoop/pull/6795#issuecomment-2112336237
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 01s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 01s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 01s | | detect-secrets was not available.
|
| +0 :ok: | shellcheck | 0m 01s | | Shellcheck was not available. |
| +0 :ok: | shelldocs | 0m 01s | | Shelldocs was not available. |
| +0 :ok: | xmllint | 0m 01s | | xmllint was not available. |
| +0 :ok: | spotbugs | 0m 01s | | spotbugs executables are not
available. |
| +0 :ok: | markdownlint | 0m 01s | | markdownlint was not available.
|
| +1 :green_heart: | @author | 0m 00s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 00s | | The patch appears to
include 1 new or modified test files. |
_ trunk Compile Tests _ |
| +0 :ok: | mvndep | 5m 00s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 89m 08s | | trunk passed |
| +1 :green_heart: | compile | 40m 19s | | trunk passed |
| +1 :green_heart: | checkstyle | 5m 57s | | trunk passed |
| -1 :x: | mvnsite | 22m 54s |
[/branch-mvnsite-root.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6795/8/artifact/out/branch-mvnsite-root.txt)
| root in trunk failed. |
| +1 :green_heart: | javadoc | 16m 09s | | trunk passed |
| +1 :green_heart: | shadedclient | 196m 27s | | branch has no errors
when building and testing our client artifacts. |
| -0 :warning: | patch | 198m 55s | | Used diff version of patch file.
Binary files and potentially other changes not applied. Please rebase and
squash commits if necessary. |
_ Patch Compile Tests _ |
| +0 :ok: | mvndep | 2m 32s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 98m 13s | | the patch passed |
| +1 :green_heart: | compile | 39m 14s | | the patch passed |
| +1 :green_heart: | javac | 39m 14s | | the patch passed |
| +1 :green_heart: | blanks | 0m 01s | | The patch has no blanks
issues. |
| +1 :green_heart: | checkstyle | 5m 56s | | the patch passed |
| -1 :x: | mvnsite | 24m 03s |
[/patch-mvnsite-root.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6795/8/artifact/out/patch-mvnsite-root.txt)
| root in the patch failed. |
| +1 :green_heart: | javadoc | 16m 39s | | the patch passed |
| +1 :green_heart: | shadedclient | 207m 32s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart: | asflicense | 6m 18s | | The patch does not
generate ASF License warnings. |
| | | 691m 15s | | |
| Subsystem | Report/Notes |
|--:|:-|
| GITHUB PR | https://github.com/apache/hadoop/pull/6795 |
| Optional Tests | dupname asflicense codespell detsecrets shellcheck
shelldocs compile javac javadoc mvninstall mvnsite unit shadedclient xmllint
spotbugs checkstyle markdownlint |
| uname | MINGW64_NT-10.0-17763 a46a48cd5fda 3.4.10-87d57229.x86_64
2024-02-14 20:17 UTC x86_64 Msys |
| Build tool | maven |
| Personality | /c/hadoop/dev-support/bin/hadoop.sh |
| git revision | trunk / 7bb4dc1b73d09c381b06c6d8460550ccadc2a2c3 |
| Default Java | Azul Systems, Inc.-1.8.0_332-b09 |
| Test Results |
https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6795/8/testReport/
|
| modules | C: hadoop-project
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site . U: . |
| Console output |
https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6795/8/console
|
| versions | git=2.45.0.windows.1 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
> FederationStateStoreFacade Cache Support Caffeine
> -
>
> Key: YARN-11471
> URL: https://issues.apache.org/jira/browse/YARN-11471
> Project: Hadoop YARN
> Is
[jira] [Commented] (YARN-11469) Refactor FederationStateStoreFacade Cache Code
[ https://issues.apache.org/jira/browse/YARN-11469?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17846504#comment-17846504 ] ASF GitHub Bot commented on YARN-11469: --- slfan1989 commented on PR #6796: URL: https://github.com/apache/hadoop/pull/6796#issuecomment-2111643363 @steveloughran @goiri We are backporting `YARN-11469` to branch-3.3 in order to upgrade ehcache from `3.3.1` to `3.8.2`. However, there are significant differences between the code in branch-3.3 and branch-3.4. I had to redevelop `YARN-11469` based on the code in branch-3.3, which has resulted in some variances between `YARN-11469` in branch-3.3 and branch-3.4, but they are generally consistent. > Refactor FederationStateStoreFacade Cache Code > -- > > Key: YARN-11469 > URL: https://issues.apache.org/jira/browse/YARN-11469 > Project: Hadoop YARN > Issue Type: Improvement > Components: federation >Affects Versions: 3.4.0 >Reporter: Shilun Fan >Assignee: Shilun Fan >Priority: Major > Labels: pull-request-available > Fix For: 3.4.0 > > > The Cache of FederationStateStoreFacade uses JCache, but considering that > JCache is not a general Cache implementation (the latest version was released > in 2014), this part of the code is refactored to support multiple Cache in > the future. -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11469) Refactor FederationStateStoreFacade Cache Code
[
https://issues.apache.org/jira/browse/YARN-11469?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17846476#comment-17846476
]
ASF GitHub Bot commented on YARN-11469:
---
hadoop-yetus commented on PR #6796:
URL: https://github.com/apache/hadoop/pull/6796#issuecomment-2111465001
:confetti_ball: **+1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 6m 25s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 1s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 0s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available.
|
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to
include 2 new or modified test files. |
_ branch-3.3 Compile Tests _ |
| +1 :green_heart: | mvninstall | 53m 43s | | branch-3.3 passed |
| +1 :green_heart: | compile | 0m 39s | | branch-3.3 passed |
| +1 :green_heart: | checkstyle | 0m 29s | | branch-3.3 passed |
| +1 :green_heart: | mvnsite | 0m 44s | | branch-3.3 passed |
| +1 :green_heart: | javadoc | 0m 40s | | branch-3.3 passed |
| +1 :green_heart: | spotbugs | 1m 25s | | branch-3.3 passed |
| +1 :green_heart: | shadedclient | 35m 24s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +1 :green_heart: | mvninstall | 0m 35s | | the patch passed |
| +1 :green_heart: | compile | 0m 31s | | the patch passed |
| +1 :green_heart: | javac | 0m 31s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks
issues. |
| +1 :green_heart: | checkstyle | 0m 18s | |
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common:
The patch generated 0 new + 0 unchanged - 11 fixed = 0 total (was 11) |
| +1 :green_heart: | mvnsite | 0m 34s | | the patch passed |
| +1 :green_heart: | javadoc | 0m 26s | |
hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common
generated 0 new + 44 unchanged - 1 fixed = 44 total (was 45) |
| +1 :green_heart: | spotbugs | 1m 22s | | the patch passed |
| +1 :green_heart: | shadedclient | 34m 49s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart: | unit | 3m 12s | | hadoop-yarn-server-common in
the patch passed. |
| +1 :green_heart: | asflicense | 0m 37s | | The patch does not
generate ASF License warnings. |
| | | 145m 34s | | |
| Subsystem | Report/Notes |
|--:|:-|
| Docker | ClientAPI=1.45 ServerAPI=1.45 base:
https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6796/4/artifact/out/Dockerfile
|
| GITHUB PR | https://github.com/apache/hadoop/pull/6796 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall
mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | Linux 945bbd036316 5.15.0-106-generic #116-Ubuntu SMP Wed Apr 17
09:17:56 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | branch-3.3 / e3cfb1a85a036ff50c99f13ac2927530f1eabc07 |
| Default Java | Private Build-1.8.0_362-8u372-ga~us1-0ubuntu1~18.04-b09 |
| Test Results |
https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6796/4/testReport/ |
| Max. process+thread count | 552 (vs. ulimit of 5500) |
| modules | C:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common U:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common |
| Console output |
https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6796/4/console |
| versions | git=2.17.1 maven=3.6.0 spotbugs=4.2.2 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
> Refactor FederationStateStoreFacade Cache Code
> --
>
> Key: YARN-11469
> URL: https://issues.apache.org/jira/browse/YARN-11469
> Project: Hadoop YARN
> Issue Type: Improvement
> Components: federation
>Affects Versions: 3.4.0
>Reporter: Shilun Fan
>Assignee: Shilun Fan
>Priority: Major
> Labels: pull-request-available
> Fix For: 3.4.0
>
>
> The Cache of FederationStateStoreFacade uses JCache, but considering t
[jira] [Commented] (YARN-11692) Support mixed cgroup v1/v2 controller structure
[
https://issues.apache.org/jira/browse/YARN-11692?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17846410#comment-17846410
]
ASF GitHub Bot commented on YARN-11692:
---
hadoop-yetus commented on PR #6821:
URL: https://github.com/apache/hadoop/pull/6821#issuecomment-2111025819
:confetti_ball: **+1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 02s | | No case conflicting files
found. |
| +0 :ok: | spotbugs | 0m 00s | | spotbugs executables are not
available. |
| +0 :ok: | codespell | 0m 00s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 00s | | detect-secrets was not available.
|
| +0 :ok: | xmllint | 0m 00s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 00s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 00s | | The patch appears to
include 2 new or modified test files. |
_ trunk Compile Tests _ |
| +0 :ok: | mvndep | 2m 19s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 90m 56s | | trunk passed |
| +1 :green_heart: | compile | 11m 49s | | trunk passed |
| +1 :green_heart: | checkstyle | 5m 19s | | trunk passed |
| +1 :green_heart: | mvnsite | 17m 11s | | trunk passed |
| +1 :green_heart: | javadoc | 15m 40s | | trunk passed |
| +1 :green_heart: | shadedclient | 176m 15s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +0 :ok: | mvndep | 2m 22s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 8m 56s | | the patch passed |
| +1 :green_heart: | compile | 9m 21s | | the patch passed |
| +1 :green_heart: | javac | 9m 21s | | the patch passed |
| +1 :green_heart: | blanks | 0m 00s | | The patch has no blanks
issues. |
| +1 :green_heart: | checkstyle | 3m 50s | | the patch passed |
| +1 :green_heart: | mvnsite | 11m 03s | | the patch passed |
| +1 :green_heart: | javadoc | 10m 44s | | the patch passed |
| +1 :green_heart: | shadedclient | 178m 40s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart: | asflicense | 5m 41s | | The patch does not
generate ASF License warnings. |
| | | 493m 47s | | |
| Subsystem | Report/Notes |
|--:|:-|
| GITHUB PR | https://github.com/apache/hadoop/pull/6821 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall
mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets xmllint |
| uname | MINGW64_NT-10.0-17763 0bed74f0c5c8 3.4.10-87d57229.x86_64
2024-02-14 20:17 UTC x86_64 Msys |
| Build tool | maven |
| Personality | /c/hadoop/dev-support/bin/hadoop.sh |
| git revision | trunk / e49e11d112734a35249a343a3c8cbb906f20eec4 |
| Default Java | Azul Systems, Inc.-1.8.0_332-b09 |
| Test Results |
https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6821/7/testReport/
|
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager
U: hadoop-yarn-project/hadoop-yarn |
| Console output |
https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6821/7/console
|
| versions | git=2.45.0.windows.1 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
> Support mixed cgroup v1/v2 controller structure
> ---
>
> Key: YARN-11692
> URL: https://issues.apache.org/jira/browse/YARN-11692
> Project: Hadoop YARN
> Issue Type: Sub-task
>Reporter: Benjamin Teke
>Assignee: Peter Szucs
>Priority: Major
> Labels: pull-request-available
>
> There were heavy changes on the device side in cgroup v2. To keep supporting
> FGPAs and GPUs short term, mixed structures where some of the cgroup
> controllers are from v1 while others from v2 should be supported. More info:
> https://dropbear.xyz/2023/05/23/devices-with-cgroup-v2/
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11692) Support mixed cgroup v1/v2 controller structure
[
https://issues.apache.org/jira/browse/YARN-11692?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17846367#comment-17846367
]
ASF GitHub Bot commented on YARN-11692:
---
hadoop-yetus commented on PR #6821:
URL: https://github.com/apache/hadoop/pull/6821#issuecomment-2110605041
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 01s | | No case conflicting files
found. |
| +0 :ok: | spotbugs | 0m 00s | | spotbugs executables are not
available. |
| +0 :ok: | codespell | 0m 00s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 00s | | detect-secrets was not available.
|
| +0 :ok: | xmllint | 0m 00s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 01s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 00s | | The patch appears to
include 2 new or modified test files. |
_ trunk Compile Tests _ |
| +0 :ok: | mvndep | 3m 43s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 85m 57s | | trunk passed |
| +1 :green_heart: | compile | 10m 53s | | trunk passed |
| +1 :green_heart: | checkstyle | 5m 04s | | trunk passed |
| +1 :green_heart: | mvnsite | 15m 22s | | trunk passed |
| +1 :green_heart: | javadoc | 14m 40s | | trunk passed |
| +1 :green_heart: | shadedclient | 165m 39s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +0 :ok: | mvndep | 2m 23s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 8m 42s | | the patch passed |
| +1 :green_heart: | compile | 9m 08s | | the patch passed |
| +1 :green_heart: | javac | 9m 08s | | the patch passed |
| -1 :x: | blanks | 0m 01s |
[/blanks-eol.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6821/6/artifact/out/blanks-eol.txt)
| The patch has 6 line(s) that end in blanks. Use git apply --whitespace=fix
<>. Refer https://git-scm.com/docs/git-apply |
| +1 :green_heart: | checkstyle | 3m 38s | | the patch passed |
| +1 :green_heart: | mvnsite | 10m 31s | | the patch passed |
| +1 :green_heart: | javadoc | 10m 07s | | the patch passed |
| +1 :green_heart: | shadedclient | 170m 26s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart: | asflicense | 5m 34s | | The patch does not
generate ASF License warnings. |
| | | 470m 12s | | |
| Subsystem | Report/Notes |
|--:|:-|
| GITHUB PR | https://github.com/apache/hadoop/pull/6821 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall
mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets xmllint |
| uname | MINGW64_NT-10.0-17763 dbf2a9ba4078 3.4.10-87d57229.x86_64
2024-02-14 20:17 UTC x86_64 Msys |
| Build tool | maven |
| Personality | /c/hadoop/dev-support/bin/hadoop.sh |
| git revision | trunk / 185d0d3a97a6d01214a3aaeb474b1d38ff8ed644 |
| Default Java | Azul Systems, Inc.-1.8.0_332-b09 |
| Test Results |
https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6821/6/testReport/
|
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager
U: hadoop-yarn-project/hadoop-yarn |
| Console output |
https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6821/6/console
|
| versions | git=2.45.0.windows.1 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
> Support mixed cgroup v1/v2 controller structure
> ---
>
> Key: YARN-11692
> URL: https://issues.apache.org/jira/browse/YARN-11692
> Project: Hadoop YARN
> Issue Type: Sub-task
>Reporter: Benjamin Teke
>Assignee: Peter Szucs
>Priority: Major
> Labels: pull-request-available
>
> There were heavy changes on the device side in cgroup v2. To keep supporting
> FGPAs and GPUs short term, mixed structures where some of the cgroup
> controllers are from v1 while others from v2 should be supported. More info:
> https://dropbear.xyz/2023/05/23/devices-with-cgroup-v2/
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---
[jira] [Commented] (YARN-11692) Support mixed cgroup v1/v2 controller structure
[
https://issues.apache.org/jira/browse/YARN-11692?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17846347#comment-17846347
]
ASF GitHub Bot commented on YARN-11692:
---
hadoop-yetus commented on PR #6821:
URL: https://github.com/apache/hadoop/pull/6821#issuecomment-2110528849
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 01s | | No case conflicting files
found. |
| +0 :ok: | spotbugs | 0m 01s | | spotbugs executables are not
available. |
| +0 :ok: | codespell | 0m 01s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 01s | | detect-secrets was not available.
|
| +0 :ok: | xmllint | 0m 01s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 00s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 00s | | The patch appears to
include 2 new or modified test files. |
_ trunk Compile Tests _ |
| +0 :ok: | mvndep | 4m 00s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 89m 29s | | trunk passed |
| +1 :green_heart: | compile | 11m 14s | | trunk passed |
| +1 :green_heart: | checkstyle | 4m 59s | | trunk passed |
| +1 :green_heart: | mvnsite | 15m 18s | | trunk passed |
| +1 :green_heart: | javadoc | 14m 54s | | trunk passed |
| +1 :green_heart: | shadedclient | 167m 45s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +0 :ok: | mvndep | 2m 24s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 8m 50s | | the patch passed |
| +1 :green_heart: | compile | 9m 17s | | the patch passed |
| +1 :green_heart: | javac | 9m 18s | | the patch passed |
| -1 :x: | blanks | 0m 00s |
[/blanks-eol.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6821/5/artifact/out/blanks-eol.txt)
| The patch has 6 line(s) that end in blanks. Use git apply --whitespace=fix
<>. Refer https://git-scm.com/docs/git-apply |
| +1 :green_heart: | checkstyle | 3m 28s | | the patch passed |
| +1 :green_heart: | mvnsite | 11m 10s | | the patch passed |
| +1 :green_heart: | javadoc | 10m 26s | | the patch passed |
| +1 :green_heart: | shadedclient | 176m 10s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart: | asflicense | 6m 09s | | The patch does not
generate ASF License warnings. |
| | | 482m 29s | | |
| Subsystem | Report/Notes |
|--:|:-|
| GITHUB PR | https://github.com/apache/hadoop/pull/6821 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall
mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets xmllint |
| uname | MINGW64_NT-10.0-17763 e05d084bc99f 3.4.10-87d57229.x86_64
2024-02-14 20:17 UTC x86_64 Msys |
| Build tool | maven |
| Personality | /c/hadoop/dev-support/bin/hadoop.sh |
| git revision | trunk / d7ffa252fc073c2d77881eecc8bfd0aa166ef967 |
| Default Java | Azul Systems, Inc.-1.8.0_332-b09 |
| Test Results |
https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6821/5/testReport/
|
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager
U: hadoop-yarn-project/hadoop-yarn |
| Console output |
https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6821/5/console
|
| versions | git=2.45.0.windows.1 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
> Support mixed cgroup v1/v2 controller structure
> ---
>
> Key: YARN-11692
> URL: https://issues.apache.org/jira/browse/YARN-11692
> Project: Hadoop YARN
> Issue Type: Sub-task
>Reporter: Benjamin Teke
>Assignee: Peter Szucs
>Priority: Major
> Labels: pull-request-available
>
> There were heavy changes on the device side in cgroup v2. To keep supporting
> FGPAs and GPUs short term, mixed structures where some of the cgroup
> controllers are from v1 while others from v2 should be supported. More info:
> https://dropbear.xyz/2023/05/23/devices-with-cgroup-v2/
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---
[jira] [Commented] (YARN-11692) Support mixed cgroup v1/v2 controller structure
[
https://issues.apache.org/jira/browse/YARN-11692?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17846346#comment-17846346
]
ASF GitHub Bot commented on YARN-11692:
---
hadoop-yetus commented on PR #6821:
URL: https://github.com/apache/hadoop/pull/6821#issuecomment-2110523072
:confetti_ball: **+1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 0m 48s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 0s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available.
|
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to
include 2 new or modified test files. |
_ trunk Compile Tests _ |
| +0 :ok: | mvndep | 14m 33s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 36m 18s | | trunk passed |
| +1 :green_heart: | compile | 8m 13s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | compile | 7m 22s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | checkstyle | 1m 56s | | trunk passed |
| +1 :green_heart: | mvnsite | 2m 44s | | trunk passed |
| +1 :green_heart: | javadoc | 2m 44s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javadoc | 2m 34s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | spotbugs | 5m 29s | | trunk passed |
| +1 :green_heart: | shadedclient | 39m 34s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 31s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 1m 45s | | the patch passed |
| +1 :green_heart: | compile | 7m 25s | | the patch passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javac | 7m 25s | | the patch passed |
| +1 :green_heart: | compile | 7m 17s | | the patch passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | javac | 7m 17s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks
issues. |
| +1 :green_heart: | checkstyle | 1m 55s | | the patch passed |
| +1 :green_heart: | mvnsite | 2m 29s | | the patch passed |
| +1 :green_heart: | javadoc | 2m 26s | | the patch passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javadoc | 2m 19s | | the patch passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | spotbugs | 5m 48s | | the patch passed |
| +1 :green_heart: | shadedclient | 40m 34s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart: | unit | 1m 4s | | hadoop-yarn-api in the patch
passed. |
| +1 :green_heart: | unit | 5m 19s | | hadoop-yarn-common in the patch
passed. |
| +1 :green_heart: | unit | 25m 6s | | hadoop-yarn-server-nodemanager
in the patch passed. |
| +1 :green_heart: | asflicense | 0m 55s | | The patch does not
generate ASF License warnings. |
| | | 233m 0s | | |
| Subsystem | Report/Notes |
|--:|:-|
| Docker | ClientAPI=1.44 ServerAPI=1.44 base:
https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6821/7/artifact/out/Dockerfile
|
| GITHUB PR | https://github.com/apache/hadoop/pull/6821 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall
mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets xmllint |
| uname | Linux 029f6a61378e 5.15.0-94-generic #104-Ubuntu SMP Tue Jan 9
15:25:40 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / e49e11d112734a35249a343a3c8cbb906f20eec4 |
| Default Java | Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| Multi-JDK versions |
/usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1
/usr/lib/jvm/java-8-openjdk-amd64:Private
Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| Test Results |
https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6821/7/testReport/ |
| Max. process+thread count
[jira] [Commented] (YARN-11692) Support mixed cgroup v1/v2 controller structure
[
https://issues.apache.org/jira/browse/YARN-11692?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17846283#comment-17846283
]
ASF GitHub Bot commented on YARN-11692:
---
hadoop-yetus commented on PR #6821:
URL: https://github.com/apache/hadoop/pull/6821#issuecomment-2110034091
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 0m 50s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 0s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available.
|
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to
include 2 new or modified test files. |
_ trunk Compile Tests _ |
| +0 :ok: | mvndep | 14m 27s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 37m 48s | | trunk passed |
| +1 :green_heart: | compile | 8m 17s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | compile | 7m 38s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | checkstyle | 2m 6s | | trunk passed |
| +1 :green_heart: | mvnsite | 2m 48s | | trunk passed |
| +1 :green_heart: | javadoc | 2m 56s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javadoc | 2m 40s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | spotbugs | 5m 44s | | trunk passed |
| +1 :green_heart: | shadedclient | 41m 18s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 32s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 1m 47s | | the patch passed |
| +1 :green_heart: | compile | 7m 37s | | the patch passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javac | 7m 37s | | the patch passed |
| +1 :green_heart: | compile | 7m 30s | | the patch passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | javac | 7m 30s | | the patch passed |
| -1 :x: | blanks | 0m 0s |
[/blanks-eol.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6821/6/artifact/out/blanks-eol.txt)
| The patch has 6 line(s) that end in blanks. Use git apply --whitespace=fix
<>. Refer https://git-scm.com/docs/git-apply |
| -0 :warning: | checkstyle | 2m 4s |
[/results-checkstyle-hadoop-yarn-project_hadoop-yarn.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6821/6/artifact/out/results-checkstyle-hadoop-yarn-project_hadoop-yarn.txt)
| hadoop-yarn-project/hadoop-yarn: The patch generated 3 new + 169 unchanged
- 0 fixed = 172 total (was 169) |
| +1 :green_heart: | mvnsite | 2m 34s | | the patch passed |
| +1 :green_heart: | javadoc | 2m 32s | | the patch passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javadoc | 2m 25s | | the patch passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | spotbugs | 6m 22s | | the patch passed |
| +1 :green_heart: | shadedclient | 40m 47s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart: | unit | 1m 8s | | hadoop-yarn-api in the patch
passed. |
| +1 :green_heart: | unit | 5m 23s | | hadoop-yarn-common in the patch
passed. |
| +1 :green_heart: | unit | 25m 15s | | hadoop-yarn-server-nodemanager
in the patch passed. |
| +1 :green_heart: | asflicense | 0m 58s | | The patch does not
generate ASF License warnings. |
| | | 239m 29s | | |
| Subsystem | Report/Notes |
|--:|:-|
| Docker | ClientAPI=1.45 ServerAPI=1.45 base:
https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6821/6/artifact/out/Dockerfile
|
| GITHUB PR | https://github.com/apache/hadoop/pull/6821 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall
mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets xmllint |
| uname | Linux 5afbe9ffa4c8 5.15.0-106-generic #116-Ubuntu SMP Wed Apr 17
09:17:56 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/had
[jira] [Commented] (YARN-11692) Support mixed cgroup v1/v2 controller structure
[
https://issues.apache.org/jira/browse/YARN-11692?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17846274#comment-17846274
]
ASF GitHub Bot commented on YARN-11692:
---
hadoop-yetus commented on PR #6821:
URL: https://github.com/apache/hadoop/pull/6821#issuecomment-2109940524
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 0m 50s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 0s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available.
|
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to
include 2 new or modified test files. |
_ trunk Compile Tests _ |
| +0 :ok: | mvndep | 14m 27s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 37m 1s | | trunk passed |
| +1 :green_heart: | compile | 8m 35s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | compile | 7m 51s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | checkstyle | 2m 4s | | trunk passed |
| +1 :green_heart: | mvnsite | 2m 49s | | trunk passed |
| +1 :green_heart: | javadoc | 2m 44s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javadoc | 2m 32s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | spotbugs | 5m 34s | | trunk passed |
| +1 :green_heart: | shadedclient | 40m 6s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 33s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 1m 58s | | the patch passed |
| +1 :green_heart: | compile | 8m 20s | | the patch passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javac | 8m 20s | | the patch passed |
| +1 :green_heart: | compile | 7m 47s | | the patch passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | javac | 7m 47s | | the patch passed |
| -1 :x: | blanks | 0m 0s |
[/blanks-eol.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6821/5/artifact/out/blanks-eol.txt)
| The patch has 6 line(s) that end in blanks. Use git apply --whitespace=fix
<>. Refer https://git-scm.com/docs/git-apply |
| -0 :warning: | checkstyle | 1m 57s |
[/results-checkstyle-hadoop-yarn-project_hadoop-yarn.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6821/5/artifact/out/results-checkstyle-hadoop-yarn-project_hadoop-yarn.txt)
| hadoop-yarn-project/hadoop-yarn: The patch generated 3 new + 169 unchanged
- 0 fixed = 172 total (was 169) |
| +1 :green_heart: | mvnsite | 2m 33s | | the patch passed |
| +1 :green_heart: | javadoc | 2m 30s | | the patch passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javadoc | 2m 19s | | the patch passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | spotbugs | 6m 3s | | the patch passed |
| +1 :green_heart: | shadedclient | 41m 2s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart: | unit | 1m 9s | | hadoop-yarn-api in the patch
passed. |
| +1 :green_heart: | unit | 5m 37s | | hadoop-yarn-common in the patch
passed. |
| +1 :green_heart: | unit | 25m 32s | | hadoop-yarn-server-nodemanager
in the patch passed. |
| +1 :green_heart: | asflicense | 0m 56s | | The patch does not
generate ASF License warnings. |
| | | 239m 1s | | |
| Subsystem | Report/Notes |
|--:|:-|
| Docker | ClientAPI=1.45 ServerAPI=1.45 base:
https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6821/5/artifact/out/Dockerfile
|
| GITHUB PR | https://github.com/apache/hadoop/pull/6821 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall
mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets xmllint |
| uname | Linux cc7c8d4f8eb0 5.15.0-106-generic #116-Ubuntu SMP Wed Apr 17
09:17:56 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/had
[jira] [Commented] (YARN-11692) Support mixed cgroup v1/v2 controller structure
[ https://issues.apache.org/jira/browse/YARN-11692?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17846207#comment-17846207 ] ASF GitHub Bot commented on YARN-11692: --- p-szucs commented on PR #6821: URL: https://github.com/apache/hadoop/pull/6821#issuecomment-2109479272 Thanks @brumi1024 for the review, I updated the patch based on your suggestions, and added unit tests as well. > Support mixed cgroup v1/v2 controller structure > --- > > Key: YARN-11692 > URL: https://issues.apache.org/jira/browse/YARN-11692 > Project: Hadoop YARN > Issue Type: Sub-task >Reporter: Benjamin Teke >Assignee: Peter Szucs >Priority: Major > Labels: pull-request-available > > There were heavy changes on the device side in cgroup v2. To keep supporting > FGPAs and GPUs short term, mixed structures where some of the cgroup > controllers are from v1 while others from v2 should be supported. More info: > https://dropbear.xyz/2023/05/23/devices-with-cgroup-v2/ -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11692) Support mixed cgroup v1/v2 controller structure
[
https://issues.apache.org/jira/browse/YARN-11692?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17846205#comment-17846205
]
ASF GitHub Bot commented on YARN-11692:
---
p-szucs commented on code in PR #6821:
URL: https://github.com/apache/hadoop/pull/6821#discussion_r1599517841
##
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java:
##
@@ -63,35 +63,54 @@ public class ResourceHandlerModule {
* as resource metrics functionality. We need to ensure that the same
* instance is used for both.
*/
+ private static volatile CGroupsHandler cGroupsV1Handler;
+ private static volatile CGroupsHandler cGroupsV2Handler;
private static volatile TrafficControlBandwidthHandlerImpl
trafficControlBandwidthHandler;
private static volatile NetworkPacketTaggingHandlerImpl
networkPacketTaggingHandlerImpl;
- private static volatile CGroupsHandler cGroupsHandler;
private static volatile CGroupsBlkioResourceHandlerImpl
cGroupsBlkioResourceHandler;
private static volatile MemoryResourceHandler
cGroupsMemoryResourceHandler;
private static volatile CpuResourceHandler
cGroupsCpuResourceHandler;
- /**
- * Returns an initialized, thread-safe CGroupsHandler instance.
- */
- private static CGroupsHandler getInitializedCGroupsHandler(Configuration
conf)
+ private static void initializeCGroupsHandlers(Configuration conf) throws
ResourceHandlerException {
+initializeCGroupsV1Handler(conf);
+if (cgroupsV2Enabled) {
+ initializeCGroupsV2Handler(conf);
+}
+ }
+
+ private static void initializeCGroupsV1Handler(Configuration conf)
throws ResourceHandlerException {
-if (cGroupsHandler == null) {
+if (cGroupsV1Handler == null) {
synchronized (CGroupsHandler.class) {
-if (cGroupsHandler == null) {
- cGroupsHandler = cgroupsV2Enabled
- ? new CGroupsV2HandlerImpl(conf,
PrivilegedOperationExecutor.getInstance(conf))
- : new CGroupsHandlerImpl(conf,
PrivilegedOperationExecutor.getInstance(conf));
- LOG.debug("Value of CGroupsHandler is: {}", cGroupsHandler);
+if (cGroupsV1Handler == null) {
Review Comment:
It catched my eye too, but then I found that it is implemented based on the
double-checked locking, so I left it there:
https://www.baeldung.com/java-singleton-double-checked-locking
> Support mixed cgroup v1/v2 controller structure
> ---
>
> Key: YARN-11692
> URL: https://issues.apache.org/jira/browse/YARN-11692
> Project: Hadoop YARN
> Issue Type: Sub-task
>Reporter: Benjamin Teke
>Assignee: Peter Szucs
>Priority: Major
> Labels: pull-request-available
>
> There were heavy changes on the device side in cgroup v2. To keep supporting
> FGPAs and GPUs short term, mixed structures where some of the cgroup
> controllers are from v1 while others from v2 should be supported. More info:
> https://dropbear.xyz/2023/05/23/devices-with-cgroup-v2/
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11692) Support mixed cgroup v1/v2 controller structure
[
https://issues.apache.org/jira/browse/YARN-11692?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17846202#comment-17846202
]
ASF GitHub Bot commented on YARN-11692:
---
p-szucs commented on code in PR #6821:
URL: https://github.com/apache/hadoop/pull/6821#discussion_r1599504396
##
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java:
##
@@ -101,18 +120,18 @@ private static CGroupsHandler
getInitializedCGroupsHandler(Configuration conf)
*/
public static CGroupsHandler getCGroupsHandler() {
-return cGroupsHandler;
+return cgroupsV2Enabled ? cGroupsV2Handler : cGroupsV1Handler;
Review Comment:
Sure, I changed it to return v1 handler only.
> Support mixed cgroup v1/v2 controller structure
> ---
>
> Key: YARN-11692
> URL: https://issues.apache.org/jira/browse/YARN-11692
> Project: Hadoop YARN
> Issue Type: Sub-task
>Reporter: Benjamin Teke
>Assignee: Peter Szucs
>Priority: Major
> Labels: pull-request-available
>
> There were heavy changes on the device side in cgroup v2. To keep supporting
> FGPAs and GPUs short term, mixed structures where some of the cgroup
> controllers are from v1 while others from v2 should be supported. More info:
> https://dropbear.xyz/2023/05/23/devices-with-cgroup-v2/
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11692) Support mixed cgroup v1/v2 controller structure
[
https://issues.apache.org/jira/browse/YARN-11692?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17846203#comment-17846203
]
ASF GitHub Bot commented on YARN-11692:
---
p-szucs commented on code in PR #6821:
URL: https://github.com/apache/hadoop/pull/6821#discussion_r1599505033
##
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java:
##
@@ -63,35 +63,54 @@ public class ResourceHandlerModule {
* as resource metrics functionality. We need to ensure that the same
* instance is used for both.
*/
+ private static volatile CGroupsHandler cGroupsV1Handler;
Review Comment:
Sure, I fixed it
##
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java:
##
@@ -63,35 +63,54 @@ public class ResourceHandlerModule {
* as resource metrics functionality. We need to ensure that the same
* instance is used for both.
*/
+ private static volatile CGroupsHandler cGroupsV1Handler;
+ private static volatile CGroupsHandler cGroupsV2Handler;
private static volatile TrafficControlBandwidthHandlerImpl
trafficControlBandwidthHandler;
private static volatile NetworkPacketTaggingHandlerImpl
networkPacketTaggingHandlerImpl;
- private static volatile CGroupsHandler cGroupsHandler;
private static volatile CGroupsBlkioResourceHandlerImpl
cGroupsBlkioResourceHandler;
private static volatile MemoryResourceHandler
cGroupsMemoryResourceHandler;
private static volatile CpuResourceHandler
cGroupsCpuResourceHandler;
- /**
- * Returns an initialized, thread-safe CGroupsHandler instance.
- */
- private static CGroupsHandler getInitializedCGroupsHandler(Configuration
conf)
+ private static void initializeCGroupsHandlers(Configuration conf) throws
ResourceHandlerException {
Review Comment:
Fixed the naming here as well.
> Support mixed cgroup v1/v2 controller structure
> ---
>
> Key: YARN-11692
> URL: https://issues.apache.org/jira/browse/YARN-11692
> Project: Hadoop YARN
> Issue Type: Sub-task
>Reporter: Benjamin Teke
>Assignee: Peter Szucs
>Priority: Major
> Labels: pull-request-available
>
> There were heavy changes on the device side in cgroup v2. To keep supporting
> FGPAs and GPUs short term, mixed structures where some of the cgroup
> controllers are from v1 while others from v2 should be supported. More info:
> https://dropbear.xyz/2023/05/23/devices-with-cgroup-v2/
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11692) Support mixed cgroup v1/v2 controller structure
[ https://issues.apache.org/jira/browse/YARN-11692?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17846201#comment-17846201 ] ASF GitHub Bot commented on YARN-11692: --- p-szucs commented on code in PR #6821: URL: https://github.com/apache/hadoop/pull/6821#discussion_r1599502676 ## hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml: ## @@ -2087,6 +2087,20 @@ yarn.nodemanager.linux-container-executor.cgroups.mount-path + +This property sets the mount path for CGroups v2. + This parameter is optional, and needed to be set only in mixed mode, + when CGroups v2 is mounted in a different path than Cgroups v1. + For example, in hybrid mode, CGroups v1 controllers can be mounted in + /sys/fs/cgroup, while v2 can be mounted in /sys/fs/cgroup/unified folder. Review Comment: It happened to be a bit misleading indeed, I fixed the phrasing. Thanks for the suggestion! > Support mixed cgroup v1/v2 controller structure > --- > > Key: YARN-11692 > URL: https://issues.apache.org/jira/browse/YARN-11692 > Project: Hadoop YARN > Issue Type: Sub-task >Reporter: Benjamin Teke >Assignee: Peter Szucs >Priority: Major > Labels: pull-request-available > > There were heavy changes on the device side in cgroup v2. To keep supporting > FGPAs and GPUs short term, mixed structures where some of the cgroup > controllers are from v1 while others from v2 should be supported. More info: > https://dropbear.xyz/2023/05/23/devices-with-cgroup-v2/ -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11692) Support mixed cgroup v1/v2 controller structure
[
https://issues.apache.org/jira/browse/YARN-11692?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17846114#comment-17846114
]
ASF GitHub Bot commented on YARN-11692:
---
hadoop-yetus commented on PR #6821:
URL: https://github.com/apache/hadoop/pull/6821#issuecomment-2109020769
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 01s | | No case conflicting files
found. |
| +0 :ok: | spotbugs | 0m 00s | | spotbugs executables are not
available. |
| +0 :ok: | codespell | 0m 00s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 00s | | detect-secrets was not available.
|
| +0 :ok: | xmllint | 0m 00s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 00s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 00s | | The patch appears to
include 1 new or modified test files. |
_ trunk Compile Tests _ |
| +0 :ok: | mvndep | 2m 26s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 89m 48s | | trunk passed |
| +1 :green_heart: | compile | 11m 29s | | trunk passed |
| +1 :green_heart: | checkstyle | 5m 19s | | trunk passed |
| +1 :green_heart: | mvnsite | 16m 31s | | trunk passed |
| +1 :green_heart: | javadoc | 16m 02s | | trunk passed |
| +1 :green_heart: | shadedclient | 172m 24s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +0 :ok: | mvndep | 2m 21s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 9m 03s | | the patch passed |
| +1 :green_heart: | compile | 9m 41s | | the patch passed |
| +1 :green_heart: | javac | 9m 41s | | the patch passed |
| -1 :x: | blanks | 0m 00s |
[/blanks-eol.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6821/3/artifact/out/blanks-eol.txt)
| The patch has 2 line(s) that end in blanks. Use git apply --whitespace=fix
<>. Refer https://git-scm.com/docs/git-apply |
| +1 :green_heart: | checkstyle | 3m 38s | | the patch passed |
| +1 :green_heart: | mvnsite | 11m 10s | | the patch passed |
| +1 :green_heart: | javadoc | 11m 09s | | the patch passed |
| +1 :green_heart: | shadedclient | 182m 23s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart: | asflicense | 5m 54s | | The patch does not
generate ASF License warnings. |
| | | 493m 14s | | |
| Subsystem | Report/Notes |
|--:|:-|
| GITHUB PR | https://github.com/apache/hadoop/pull/6821 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall
mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets xmllint |
| uname | MINGW64_NT-10.0-17763 c44c10826a47 3.4.10-87d57229.x86_64
2024-02-14 20:17 UTC x86_64 Msys |
| Build tool | maven |
| Personality | /c/hadoop/dev-support/bin/hadoop.sh |
| git revision | trunk / 280cac08b178d8b844f9309c6ad16d7f868320e6 |
| Default Java | Azul Systems, Inc.-1.8.0_332-b09 |
| Test Results |
https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6821/3/testReport/
|
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager
U: hadoop-yarn-project/hadoop-yarn |
| Console output |
https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6821/3/console
|
| versions | git=2.45.0.windows.1 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
> Support mixed cgroup v1/v2 controller structure
> ---
>
> Key: YARN-11692
> URL: https://issues.apache.org/jira/browse/YARN-11692
> Project: Hadoop YARN
> Issue Type: Sub-task
>Reporter: Benjamin Teke
>Assignee: Peter Szucs
>Priority: Major
> Labels: pull-request-available
>
> There were heavy changes on the device side in cgroup v2. To keep supporting
> FGPAs and GPUs short term, mixed structures where some of the cgroup
> controllers are from v1 while others from v2 should be supported. More info:
> https://dropbear.xyz/2023/05/23/devices-with-cgroup-v2/
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---
[jira] [Commented] (YARN-11687) Update CGroupsResourceCalculator to track usages using cgroupv2
[
https://issues.apache.org/jira/browse/YARN-11687?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17846113#comment-17846113
]
ASF GitHub Bot commented on YARN-11687:
---
hadoop-yetus commented on PR #6780:
URL: https://github.com/apache/hadoop/pull/6780#issuecomment-2109018378
:confetti_ball: **+1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 01s | | No case conflicting files
found. |
| +0 :ok: | spotbugs | 0m 01s | | spotbugs executables are not
available. |
| +0 :ok: | codespell | 0m 01s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 01s | | detect-secrets was not available.
|
| +1 :green_heart: | @author | 0m 00s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 00s | | The patch appears to
include 2 new or modified test files. |
_ trunk Compile Tests _ |
| +0 :ok: | mvndep | 2m 23s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 90m 56s | | trunk passed |
| +1 :green_heart: | compile | 11m 42s | | trunk passed |
| +1 :green_heart: | checkstyle | 5m 26s | | trunk passed |
| +1 :green_heart: | mvnsite | 11m 04s | | trunk passed |
| +1 :green_heart: | javadoc | 11m 05s | | trunk passed |
| +1 :green_heart: | shadedclient | 165m 55s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +0 :ok: | mvndep | 2m 22s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 6m 02s | | the patch passed |
| +1 :green_heart: | compile | 9m 34s | | the patch passed |
| +1 :green_heart: | javac | 9m 34s | | the patch passed |
| +1 :green_heart: | blanks | 0m 01s | | The patch has no blanks
issues. |
| +1 :green_heart: | checkstyle | 3m 40s | | the patch passed |
| +1 :green_heart: | mvnsite | 7m 24s | | the patch passed |
| +1 :green_heart: | javadoc | 7m 40s | | the patch passed |
| +1 :green_heart: | shadedclient | 172m 12s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart: | asflicense | 5m 43s | | The patch does not
generate ASF License warnings. |
| | | 474m 14s | | |
| Subsystem | Report/Notes |
|--:|:-|
| GITHUB PR | https://github.com/apache/hadoop/pull/6780 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall
mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | MINGW64_NT-10.0-17763 c7404ad146ae 3.4.10-87d57229.x86_64
2024-02-14 20:17 UTC x86_64 Msys |
| Build tool | maven |
| Personality | /c/hadoop/dev-support/bin/hadoop.sh |
| git revision | trunk / e6e9943de32b64cca6082051364d8eda099867af |
| Default Java | Azul Systems, Inc.-1.8.0_332-b09 |
| Test Results |
https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6780/3/testReport/
|
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager
U: hadoop-yarn-project/hadoop-yarn |
| Console output |
https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6780/3/console
|
| versions | git=2.44.0.windows.1 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
> Update CGroupsResourceCalculator to track usages using cgroupv2
> ---
>
> Key: YARN-11687
> URL: https://issues.apache.org/jira/browse/YARN-11687
> Project: Hadoop YARN
> Issue Type: Sub-task
>Reporter: Benjamin Teke
>Assignee: Bence Kosztolnik
>Priority: Major
> Labels: pull-request-available
>
> [CGroupsResourceCalculator|https://github.com/apache/hadoop/blob/f609460bda0c2bd87dd3580158e549e2f34f14d5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java]
> should also be updated to handle the cgroup v2 changes.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11692) Support mixed cgroup v1/v2 controller structure
[
https://issues.apache.org/jira/browse/YARN-11692?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17846103#comment-17846103
]
ASF GitHub Bot commented on YARN-11692:
---
hadoop-yetus commented on PR #6821:
URL: https://github.com/apache/hadoop/pull/6821#issuecomment-2108904702
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 0m 53s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 0s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available.
|
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to
include 2 new or modified test files. |
_ trunk Compile Tests _ |
| +0 :ok: | mvndep | 15m 17s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 36m 57s | | trunk passed |
| +1 :green_heart: | compile | 8m 20s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | compile | 7m 37s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | checkstyle | 2m 4s | | trunk passed |
| +1 :green_heart: | mvnsite | 2m 50s | | trunk passed |
| +1 :green_heart: | javadoc | 2m 52s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javadoc | 2m 40s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | spotbugs | 5m 39s | | trunk passed |
| +1 :green_heart: | shadedclient | 40m 43s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 33s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 1m 48s | | the patch passed |
| +1 :green_heart: | compile | 7m 38s | | the patch passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javac | 7m 38s | | the patch passed |
| +1 :green_heart: | compile | 7m 34s | | the patch passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | javac | 7m 34s | | the patch passed |
| -1 :x: | blanks | 0m 0s |
[/blanks-eol.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6821/4/artifact/out/blanks-eol.txt)
| The patch has 8 line(s) that end in blanks. Use git apply --whitespace=fix
<>. Refer https://git-scm.com/docs/git-apply |
| -0 :warning: | checkstyle | 1m 57s |
[/results-checkstyle-hadoop-yarn-project_hadoop-yarn.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6821/4/artifact/out/results-checkstyle-hadoop-yarn-project_hadoop-yarn.txt)
| hadoop-yarn-project/hadoop-yarn: The patch generated 5 new + 169 unchanged
- 0 fixed = 174 total (was 169) |
| +1 :green_heart: | mvnsite | 2m 38s | | the patch passed |
| +1 :green_heart: | javadoc | 2m 37s | | the patch passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javadoc | 2m 24s | | the patch passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | spotbugs | 5m 57s | | the patch passed |
| +1 :green_heart: | shadedclient | 41m 6s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart: | unit | 1m 9s | | hadoop-yarn-api in the patch
passed. |
| +1 :green_heart: | unit | 5m 23s | | hadoop-yarn-common in the patch
passed. |
| +1 :green_heart: | unit | 25m 15s | | hadoop-yarn-server-nodemanager
in the patch passed. |
| +1 :green_heart: | asflicense | 0m 58s | | The patch does not
generate ASF License warnings. |
| | | 239m 50s | | |
| Subsystem | Report/Notes |
|--:|:-|
| Docker | ClientAPI=1.45 ServerAPI=1.45 base:
https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6821/4/artifact/out/Dockerfile
|
| GITHUB PR | https://github.com/apache/hadoop/pull/6821 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall
mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets xmllint |
| uname | Linux 7cd411d3fb51 5.15.0-106-generic #116-Ubuntu SMP Wed Apr 17
09:17:56 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/had
[jira] [Commented] (YARN-11687) Update CGroupsResourceCalculator to track usages using cgroupv2
[
https://issues.apache.org/jira/browse/YARN-11687?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17846088#comment-17846088
]
ASF GitHub Bot commented on YARN-11687:
---
hadoop-yetus commented on PR #6780:
URL: https://github.com/apache/hadoop/pull/6780#issuecomment-2108868944
:confetti_ball: **+1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 01s | | No case conflicting files
found. |
| +0 :ok: | spotbugs | 0m 01s | | spotbugs executables are not
available. |
| +0 :ok: | codespell | 0m 01s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 01s | | detect-secrets was not available.
|
| +1 :green_heart: | @author | 0m 00s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 00s | | The patch appears to
include 2 new or modified test files. |
_ trunk Compile Tests _ |
| +0 :ok: | mvndep | 2m 26s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 93m 01s | | trunk passed |
| +1 :green_heart: | compile | 12m 17s | | trunk passed |
| +1 :green_heart: | checkstyle | 5m 30s | | trunk passed |
| +1 :green_heart: | mvnsite | 11m 34s | | trunk passed |
| +1 :green_heart: | javadoc | 11m 04s | | trunk passed |
| +1 :green_heart: | shadedclient | 167m 30s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +0 :ok: | mvndep | 2m 37s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 6m 18s | | the patch passed |
| +1 :green_heart: | compile | 10m 20s | | the patch passed |
| +1 :green_heart: | javac | 10m 20s | | the patch passed |
| +1 :green_heart: | blanks | 0m 00s | | The patch has no blanks
issues. |
| +1 :green_heart: | checkstyle | 3m 44s | | the patch passed |
| +1 :green_heart: | mvnsite | 8m 10s | | the patch passed |
| +1 :green_heart: | javadoc | 7m 28s | | the patch passed |
| +1 :green_heart: | shadedclient | 177m 28s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart: | asflicense | 5m 49s | | The patch does not
generate ASF License warnings. |
| | | 485m 35s | | |
| Subsystem | Report/Notes |
|--:|:-|
| GITHUB PR | https://github.com/apache/hadoop/pull/6780 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall
mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | MINGW64_NT-10.0-17763 7fe4d9bd8e01 3.4.10-87d57229.x86_64
2024-02-14 20:17 UTC x86_64 Msys |
| Build tool | maven |
| Personality | /c/hadoop/dev-support/bin/hadoop.sh |
| git revision | trunk / c84a449fc4bd9d3b62a51bf3145b54a6b422fe01 |
| Default Java | Azul Systems, Inc.-1.8.0_332-b09 |
| Test Results |
https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6780/2/testReport/
|
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager
U: hadoop-yarn-project/hadoop-yarn |
| Console output |
https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6780/2/console
|
| versions | git=2.45.0.windows.1 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
> Update CGroupsResourceCalculator to track usages using cgroupv2
> ---
>
> Key: YARN-11687
> URL: https://issues.apache.org/jira/browse/YARN-11687
> Project: Hadoop YARN
> Issue Type: Sub-task
>Reporter: Benjamin Teke
>Assignee: Bence Kosztolnik
>Priority: Major
> Labels: pull-request-available
>
> [CGroupsResourceCalculator|https://github.com/apache/hadoop/blob/f609460bda0c2bd87dd3580158e549e2f34f14d5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java]
> should also be updated to handle the cgroup v2 changes.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11692) Support mixed cgroup v1/v2 controller structure
[
https://issues.apache.org/jira/browse/YARN-11692?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17846038#comment-17846038
]
ASF GitHub Bot commented on YARN-11692:
---
hadoop-yetus commented on PR #6821:
URL: https://github.com/apache/hadoop/pull/6821#issuecomment-2108554141
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 0m 48s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 1s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 1s | | detect-secrets was not available.
|
| +0 :ok: | xmllint | 0m 1s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to
include 1 new or modified test files. |
_ trunk Compile Tests _ |
| +0 :ok: | mvndep | 14m 57s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 38m 56s | | trunk passed |
| +1 :green_heart: | compile | 8m 22s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | compile | 7m 34s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | checkstyle | 2m 3s | | trunk passed |
| +1 :green_heart: | mvnsite | 2m 52s | | trunk passed |
| +1 :green_heart: | javadoc | 2m 50s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javadoc | 2m 45s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | spotbugs | 5m 38s | | trunk passed |
| +1 :green_heart: | shadedclient | 39m 58s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 33s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 1m 49s | | the patch passed |
| +1 :green_heart: | compile | 7m 38s | | the patch passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javac | 7m 38s | | the patch passed |
| +1 :green_heart: | compile | 7m 28s | | the patch passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | javac | 7m 28s | | the patch passed |
| -1 :x: | blanks | 0m 0s |
[/blanks-eol.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6821/3/artifact/out/blanks-eol.txt)
| The patch has 2 line(s) that end in blanks. Use git apply --whitespace=fix
<>. Refer https://git-scm.com/docs/git-apply |
| -0 :warning: | checkstyle | 1m 56s |
[/results-checkstyle-hadoop-yarn-project_hadoop-yarn.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6821/3/artifact/out/results-checkstyle-hadoop-yarn-project_hadoop-yarn.txt)
| hadoop-yarn-project/hadoop-yarn: The patch generated 2 new + 165 unchanged
- 0 fixed = 167 total (was 165) |
| +1 :green_heart: | mvnsite | 2m 37s | | the patch passed |
| +1 :green_heart: | javadoc | 2m 35s | | the patch passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javadoc | 2m 28s | | the patch passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | spotbugs | 6m 0s | | the patch passed |
| +1 :green_heart: | shadedclient | 40m 21s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart: | unit | 1m 9s | | hadoop-yarn-api in the patch
passed. |
| +1 :green_heart: | unit | 5m 22s | | hadoop-yarn-common in the patch
passed. |
| +1 :green_heart: | unit | 25m 26s | | hadoop-yarn-server-nodemanager
in the patch passed. |
| +1 :green_heart: | asflicense | 0m 59s | | The patch does not
generate ASF License warnings. |
| | | 239m 39s | | |
| Subsystem | Report/Notes |
|--:|:-|
| Docker | ClientAPI=1.45 ServerAPI=1.45 base:
https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6821/3/artifact/out/Dockerfile
|
| GITHUB PR | https://github.com/apache/hadoop/pull/6821 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall
mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets xmllint |
| uname | Linux 7934c122be43 5.15.0-106-generic #116-Ubuntu SMP Wed Apr 17
09:17:56 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/had
[jira] [Commented] (YARN-11687) Update CGroupsResourceCalculator to track usages using cgroupv2
[
https://issues.apache.org/jira/browse/YARN-11687?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17846034#comment-17846034
]
ASF GitHub Bot commented on YARN-11687:
---
hadoop-yetus commented on PR #6780:
URL: https://github.com/apache/hadoop/pull/6780#issuecomment-2108463869
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 0m 23s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 1s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 1s | | detect-secrets was not available.
|
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to
include 2 new or modified test files. |
_ trunk Compile Tests _ |
| +0 :ok: | mvndep | 14m 8s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 28m 14s | | trunk passed |
| +1 :green_heart: | compile | 4m 47s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | compile | 4m 26s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | checkstyle | 1m 12s | | trunk passed |
| +1 :green_heart: | mvnsite | 1m 20s | | trunk passed |
| +1 :green_heart: | javadoc | 1m 21s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javadoc | 1m 9s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | spotbugs | 2m 27s | | trunk passed |
| +1 :green_heart: | shadedclient | 28m 16s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 24s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 0m 51s | | the patch passed |
| +1 :green_heart: | compile | 4m 31s | | the patch passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javac | 4m 31s | | the patch passed |
| +1 :green_heart: | compile | 4m 26s | | the patch passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | javac | 4m 26s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks
issues. |
| -0 :warning: | checkstyle | 1m 4s |
[/results-checkstyle-hadoop-yarn-project_hadoop-yarn.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6780/13/artifact/out/results-checkstyle-hadoop-yarn-project_hadoop-yarn.txt)
| hadoop-yarn-project/hadoop-yarn: The patch generated 12 new + 3 unchanged -
0 fixed = 15 total (was 3) |
| +1 :green_heart: | mvnsite | 1m 9s | | the patch passed |
| -1 :x: | javadoc | 0m 30s |
[/results-javadoc-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6780/13/artifact/out/results-javadoc-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt)
|
hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1
with JDK Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 generated 1 new + 98
unchanged - 1 fixed = 99 total (was 99) |
| +1 :green_heart: | javadoc | 0m 33s | | hadoop-yarn-common in the
patch passed with JDK Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06. |
| +1 :green_heart: | javadoc | 0m 32s | |
hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager-jdkPrivateBuild-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06
with JDK Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 generated 0 new +
195 unchanged - 1 fixed = 195 total (was 196) |
| -1 :x: | spotbugs | 1m 10s |
[/new-spotbugs-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager.html](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6780/13/artifact/out/new-spotbugs-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager.html)
|
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager
generated 2 new + 0 unchanged - 0 fixed = 2 total (was 0) |
| +1 :green_heart: | shadedclient | 28m 36s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1
[jira] [Commented] (YARN-11687) Update CGroupsResourceCalculator to track usages using cgroupv2
[
https://issues.apache.org/jira/browse/YARN-11687?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17846020#comment-17846020
]
ASF GitHub Bot commented on YARN-11687:
---
hadoop-yetus commented on PR #6780:
URL: https://github.com/apache/hadoop/pull/6780#issuecomment-2108294132
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 1m 4s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 2s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 2s | | detect-secrets was not available.
|
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to
include 2 new or modified test files. |
_ trunk Compile Tests _ |
| +0 :ok: | mvndep | 14m 54s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 35m 35s | | trunk passed |
| +1 :green_heart: | compile | 7m 40s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | compile | 7m 22s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | checkstyle | 2m 0s | | trunk passed |
| +1 :green_heart: | mvnsite | 1m 52s | | trunk passed |
| +1 :green_heart: | javadoc | 2m 2s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javadoc | 1m 57s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | spotbugs | 3m 32s | | trunk passed |
| +1 :green_heart: | shadedclient | 34m 15s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 35s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 1m 9s | | the patch passed |
| +1 :green_heart: | compile | 6m 51s | | the patch passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javac | 6m 51s | | the patch passed |
| +1 :green_heart: | compile | 6m 53s | | the patch passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | javac | 6m 53s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks
issues. |
| -0 :warning: | checkstyle | 1m 50s |
[/results-checkstyle-hadoop-yarn-project_hadoop-yarn.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6780/12/artifact/out/results-checkstyle-hadoop-yarn-project_hadoop-yarn.txt)
| hadoop-yarn-project/hadoop-yarn: The patch generated 12 new + 3 unchanged -
0 fixed = 15 total (was 3) |
| +1 :green_heart: | mvnsite | 1m 45s | | the patch passed |
| -1 :x: | javadoc | 0m 49s |
[/results-javadoc-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6780/12/artifact/out/results-javadoc-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1.txt)
|
hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager-jdkUbuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1
with JDK Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 generated 1 new + 98
unchanged - 1 fixed = 99 total (was 99) |
| +1 :green_heart: | javadoc | 0m 56s | | hadoop-yarn-common in the
patch passed with JDK Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06. |
| +1 :green_heart: | javadoc | 0m 49s | |
hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager-jdkPrivateBuild-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06
with JDK Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 generated 0 new +
195 unchanged - 1 fixed = 195 total (was 196) |
| -1 :x: | spotbugs | 1m 45s |
[/new-spotbugs-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager.html](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6780/12/artifact/out/new-spotbugs-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager.html)
|
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager
generated 2 new + 0 unchanged - 0 fixed = 2 total (was 0) |
| +1 :green_heart: | shadedclient | 34m 28s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1
[jira] [Commented] (YARN-11692) Support mixed cgroup v1/v2 controller structure
[
https://issues.apache.org/jira/browse/YARN-11692?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17846009#comment-17846009
]
ASF GitHub Bot commented on YARN-11692:
---
hadoop-yetus commented on PR #6821:
URL: https://github.com/apache/hadoop/pull/6821#issuecomment-2108203238
:confetti_ball: **+1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 01s | | No case conflicting files
found. |
| +0 :ok: | spotbugs | 0m 00s | | spotbugs executables are not
available. |
| +0 :ok: | codespell | 0m 01s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 01s | | detect-secrets was not available.
|
| +1 :green_heart: | @author | 0m 00s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 00s | | The patch appears to
include 1 new or modified test files. |
_ trunk Compile Tests _ |
| +1 :green_heart: | mvninstall | 85m 29s | | trunk passed |
| +1 :green_heart: | compile | 5m 10s | | trunk passed |
| +1 :green_heart: | checkstyle | 4m 30s | | trunk passed |
| +1 :green_heart: | mvnsite | 5m 06s | | trunk passed |
| +1 :green_heart: | javadoc | 4m 52s | | trunk passed |
| +1 :green_heart: | shadedclient | 140m 52s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +1 :green_heart: | mvninstall | 2m 48s | | the patch passed |
| +1 :green_heart: | compile | 2m 41s | | the patch passed |
| +1 :green_heart: | javac | 2m 41s | | the patch passed |
| +1 :green_heart: | blanks | 0m 01s | | The patch has no blanks
issues. |
| +1 :green_heart: | checkstyle | 2m 07s | | the patch passed |
| +1 :green_heart: | mvnsite | 2m 40s | | the patch passed |
| +1 :green_heart: | javadoc | 2m 29s | | the patch passed |
| +1 :green_heart: | shadedclient | 153m 08s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart: | asflicense | 6m 03s | | The patch does not
generate ASF License warnings. |
| | | 404m 29s | | |
| Subsystem | Report/Notes |
|--:|:-|
| GITHUB PR | https://github.com/apache/hadoop/pull/6821 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall
mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | MINGW64_NT-10.0-17763 df1f1c70bad4 3.4.10-87d57229.x86_64
2024-02-14 20:17 UTC x86_64 Msys |
| Build tool | maven |
| Personality | /c/hadoop/dev-support/bin/hadoop.sh |
| git revision | trunk / d945e4e08428ca2ad25c524e55ce5afd990a055a |
| Default Java | Azul Systems, Inc.-1.8.0_332-b09 |
| Test Results |
https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6821/1/testReport/
|
| modules | C:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager
U:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager
|
| Console output |
https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6821/1/console
|
| versions | git=2.44.0.windows.1 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
> Support mixed cgroup v1/v2 controller structure
> ---
>
> Key: YARN-11692
> URL: https://issues.apache.org/jira/browse/YARN-11692
> Project: Hadoop YARN
> Issue Type: Sub-task
>Reporter: Benjamin Teke
>Assignee: Peter Szucs
>Priority: Major
> Labels: pull-request-available
>
> There were heavy changes on the device side in cgroup v2. To keep supporting
> FGPAs and GPUs short term, mixed structures where some of the cgroup
> controllers are from v1 while others from v2 should be supported. More info:
> https://dropbear.xyz/2023/05/23/devices-with-cgroup-v2/
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11471) FederationStateStoreFacade Cache Support Caffeine
[ https://issues.apache.org/jira/browse/YARN-11471?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17845998#comment-17845998 ] ASF GitHub Bot commented on YARN-11471: --- hadoop-yetus commented on PR #6795: URL: https://github.com/apache/hadoop/pull/6795#issuecomment-2108107635 :broken_heart: **-1 overall** | Vote | Subsystem | Runtime | Logfile | Comment | |::|--:|:|::|:---:| | -1 :x: | patch | 0m 44s | | https://github.com/apache/hadoop/pull/6795 does not apply to trunk. Rebase required? Wrong Branch? See https://cwiki.apache.org/confluence/display/HADOOP/How+To+Contribute for help. | | Subsystem | Report/Notes | |--:|:-| | GITHUB PR | https://github.com/apache/hadoop/pull/6795 | | Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6795/7/console | | versions | git=2.45.0.windows.1 | | Powered by | Apache Yetus 0.14.0 https://yetus.apache.org | This message was automatically generated. > FederationStateStoreFacade Cache Support Caffeine > - > > Key: YARN-11471 > URL: https://issues.apache.org/jira/browse/YARN-11471 > Project: Hadoop YARN > Issue Type: Improvement > Components: federation >Affects Versions: 3.4.0 >Reporter: Shilun Fan >Assignee: Shilun Fan >Priority: Major > Labels: pull-request-available > > FederationStateStoreFacade Cache Support Caffeine -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11471) FederationStateStoreFacade Cache Support Caffeine
[ https://issues.apache.org/jira/browse/YARN-11471?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17845982#comment-17845982 ] ASF GitHub Bot commented on YARN-11471: --- hadoop-yetus commented on PR #6795: URL: https://github.com/apache/hadoop/pull/6795#issuecomment-2107954804 :broken_heart: **-1 overall** | Vote | Subsystem | Runtime | Logfile | Comment | |::|--:|:|::|:---:| | +0 :ok: | reexec | 0m 0s | | Docker mode activated. | | -1 :x: | patch | 0m 21s | | https://github.com/apache/hadoop/pull/6795 does not apply to trunk. Rebase required? Wrong Branch? See https://cwiki.apache.org/confluence/display/HADOOP/How+To+Contribute for help. | | Subsystem | Report/Notes | |--:|:-| | GITHUB PR | https://github.com/apache/hadoop/pull/6795 | | Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6795/7/console | | versions | git=2.34.1 | | Powered by | Apache Yetus 0.14.0 https://yetus.apache.org | This message was automatically generated. > FederationStateStoreFacade Cache Support Caffeine > - > > Key: YARN-11471 > URL: https://issues.apache.org/jira/browse/YARN-11471 > Project: Hadoop YARN > Issue Type: Improvement > Components: federation >Affects Versions: 3.4.0 >Reporter: Shilun Fan >Assignee: Shilun Fan >Priority: Major > Labels: pull-request-available > > FederationStateStoreFacade Cache Support Caffeine -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11471) FederationStateStoreFacade Cache Support Caffeine
[ https://issues.apache.org/jira/browse/YARN-11471?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17845978#comment-17845978 ] ASF GitHub Bot commented on YARN-11471: --- hadoop-yetus commented on PR #6795: URL: https://github.com/apache/hadoop/pull/6795#issuecomment-2107894913 :broken_heart: **-1 overall** | Vote | Subsystem | Runtime | Logfile | Comment | |::|--:|:|::|:---:| | +0 :ok: | reexec | 0m 0s | | Docker mode activated. | | -1 :x: | patch | 0m 19s | | https://github.com/apache/hadoop/pull/6795 does not apply to trunk. Rebase required? Wrong Branch? See https://cwiki.apache.org/confluence/display/HADOOP/How+To+Contribute for help. | | Subsystem | Report/Notes | |--:|:-| | GITHUB PR | https://github.com/apache/hadoop/pull/6795 | | Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6795/6/console | | versions | git=2.34.1 | | Powered by | Apache Yetus 0.14.0 https://yetus.apache.org | This message was automatically generated. > FederationStateStoreFacade Cache Support Caffeine > - > > Key: YARN-11471 > URL: https://issues.apache.org/jira/browse/YARN-11471 > Project: Hadoop YARN > Issue Type: Improvement > Components: federation >Affects Versions: 3.4.0 >Reporter: Shilun Fan >Assignee: Shilun Fan >Priority: Major > Labels: pull-request-available > > FederationStateStoreFacade Cache Support Caffeine -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11687) Update CGroupsResourceCalculator to track usages using cgroupv2
[
https://issues.apache.org/jira/browse/YARN-11687?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17845970#comment-17845970
]
ASF GitHub Bot commented on YARN-11687:
---
brumi1024 commented on code in PR #6780:
URL: https://github.com/apache/hadoop/pull/6780#discussion_r1598604638
##
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/AbstractCGroupsResourceCalculator.java:
##
@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package
org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
+
+import java.io.IOException;
+import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.util.CpuTimeTracker;
+import org.apache.hadoop.util.SysInfoLinux;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
+import org.apache.hadoop.yarn.util.SystemClock;
+
+/**
+ * Common code base for the CGroupsResourceCalculator implementations.
+ */
+public abstract class AbstractCGroupsResourceCalculator extends
ResourceCalculatorProcessTree {
+ private static final Logger LOG =
LoggerFactory.getLogger(AbstractCGroupsResourceCalculator.class);
+ protected final String pid;
+ protected final Clock clock = SystemClock.getInstance();
+ protected final Map stats = new ConcurrentHashMap<>();
+
+ @VisibleForTesting
+ long jiffyLengthMs = SysInfoLinux.JIFFY_LENGTH_IN_MILLIS;
+ @VisibleForTesting
+ CpuTimeTracker cpuTimeTracker;
+ @VisibleForTesting
+ CGroupsHandler cGroupsHandler;
+ @VisibleForTesting
+ String root = "/";
Review Comment:
Thanks! I agree with the points above, resolving this.
> Update CGroupsResourceCalculator to track usages using cgroupv2
> ---
>
> Key: YARN-11687
> URL: https://issues.apache.org/jira/browse/YARN-11687
> Project: Hadoop YARN
> Issue Type: Sub-task
>Reporter: Benjamin Teke
>Assignee: Bence Kosztolnik
>Priority: Major
> Labels: pull-request-available
>
> [CGroupsResourceCalculator|https://github.com/apache/hadoop/blob/f609460bda0c2bd87dd3580158e549e2f34f14d5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java]
> should also be updated to handle the cgroup v2 changes.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11687) Update CGroupsResourceCalculator to track usages using cgroupv2
[
https://issues.apache.org/jira/browse/YARN-11687?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17845969#comment-17845969
]
ASF GitHub Bot commented on YARN-11687:
---
brumi1024 commented on code in PR #6780:
URL: https://github.com/apache/hadoop/pull/6780#discussion_r1598585675
##
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java:
##
@@ -18,38 +18,20 @@
package
org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
-import org.apache.hadoop.classification.VisibleForTesting;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.util.CpuTimeTracker;
-import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.util.SysInfoLinux;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.util.Clock;
-import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
-import org.apache.hadoop.yarn.util.SystemClock;
-
-import java.io.BufferedReader;
import java.io.File;
-import java.io.FileInputStream;
import java.io.IOException;
-import java.io.InputStreamReader;
-import java.math.BigInteger;
-import java.nio.charset.StandardCharsets;
-import java.util.function.Function;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
/**
* A cgroups file-system based Resource calculator without the process tree
* features.
*
- * CGroups has its limitations. It can only be enabled, if both CPU and memory
- * cgroups are enabled with yarn.nodemanager.resource.cpu.enabled and
- * yarn.nodemanager.resource.memory.enabled respectively. This means that
- * memory limits are enforced by default. You can turn this off and keep
- * memory reporting only with yarn.nodemanager.resource.memory.enforced.
Review Comment:
Was this removed by mistake, or is this limitation not true anymore? If it's
no longer relevant the other limitation shouldn't start with "Another
limitation".
##
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/AbstractCGroupsResourceCalculator.java:
##
@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package
org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
+
+import java.io.IOException;
+import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.util.CpuTimeTracker;
+import org.apache.hadoop.util.SysInfoLinux;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
+import org.apache.hadoop.yarn.util.SystemClock;
+
+/**
+ * Common code base for the CGroupsResourceCalculator implementations.
+ */
+public abstract class AbstractCGroupsResourceCalculator extends
ResourceCalculatorProcessTree {
+ private static final Logger LOG =
LoggerFactory.getLogger(AbstractCGroupsResourceCalculator.class);
+ protected final String pid;
+ protected final Clock clock = SystemClock.getInstance();
+ protected final Map stats = new ConcurrentHashMap<>();
+
+ @VisibleForTesting
+ long jiffyLengthMs = SysInfoLinux.JIFFY_LENGTH_IN_MILLIS;
+ @VisibleForTesting
+ CpuTimeTracker cpuTimeTracker;
+ @VisibleForTesting
+ CGroupsHandler cGroupsHandler;
+ @VisibleForTesting
+ String procFs
[jira] [Commented] (YARN-11687) Update CGroupsResourceCalculator to track usages using cgroupv2
[
https://issues.apache.org/jira/browse/YARN-11687?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17845932#comment-17845932
]
ASF GitHub Bot commented on YARN-11687:
---
K0K0V0K commented on code in PR #6780:
URL: https://github.com/apache/hadoop/pull/6780#discussion_r1598513673
##
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/AbstractCGroupsResourceCalculator.java:
##
@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package
org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
+
+import java.io.IOException;
+import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.util.CpuTimeTracker;
+import org.apache.hadoop.util.SysInfoLinux;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
+import org.apache.hadoop.yarn.util.SystemClock;
+
+/**
+ * Common code base for the CGroupsResourceCalculator implementations.
+ */
+public abstract class AbstractCGroupsResourceCalculator extends
ResourceCalculatorProcessTree {
+ private static final Logger LOG =
LoggerFactory.getLogger(AbstractCGroupsResourceCalculator.class);
+ protected final String pid;
+ protected final Clock clock = SystemClock.getInstance();
+ protected final Map stats = new ConcurrentHashMap<>();
+
+ @VisibleForTesting
+ long jiffyLengthMs = SysInfoLinux.JIFFY_LENGTH_IN_MILLIS;
+ @VisibleForTesting
+ CpuTimeTracker cpuTimeTracker;
+ @VisibleForTesting
+ CGroupsHandler cGroupsHandler;
+ @VisibleForTesting
+ String root = "/";
Review Comment:
Thanks for the review!
I modified this variable to procFs. However, I would rather keep this out of
the constructor.
- If we create a new constructor with this "VisibleForTesting" annotation we
will have a 4 and 5 params constructors, which seems a bit heavy to me.
- To use only one method to set param for testing, I should add the other 3
params (jiffyLengthMs, cpuTimeTracker, cGroupsHandler) to the new constructor,
so actually we will have a 4 and 8 param constructor.
- if we move the PROCFS variable to the ResourceCalculatorProcessTree, it
will be present in the WindowsBasedProcessTree which uses a shell command to
get this information and the variable can be misleading in there context.
> Update CGroupsResourceCalculator to track usages using cgroupv2
> ---
>
> Key: YARN-11687
> URL: https://issues.apache.org/jira/browse/YARN-11687
> Project: Hadoop YARN
> Issue Type: Sub-task
>Reporter: Benjamin Teke
>Assignee: Bence Kosztolnik
>Priority: Major
> Labels: pull-request-available
>
> [CGroupsResourceCalculator|https://github.com/apache/hadoop/blob/f609460bda0c2bd87dd3580158e549e2f34f14d5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java]
> should also be updated to handle the cgroup v2 changes.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11692) Support mixed cgroup v1/v2 controller structure
[
https://issues.apache.org/jira/browse/YARN-11692?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17845919#comment-17845919
]
ASF GitHub Bot commented on YARN-11692:
---
hadoop-yetus commented on PR #6821:
URL: https://github.com/apache/hadoop/pull/6821#issuecomment-2107539816
:confetti_ball: **+1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 18m 36s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 0s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available.
|
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to
include 1 new or modified test files. |
_ trunk Compile Tests _ |
| +1 :green_heart: | mvninstall | 51m 29s | | trunk passed |
| +1 :green_heart: | compile | 1m 28s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | compile | 1m 23s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | checkstyle | 0m 37s | | trunk passed |
| +1 :green_heart: | mvnsite | 0m 43s | | trunk passed |
| +1 :green_heart: | javadoc | 0m 46s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javadoc | 0m 39s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | spotbugs | 1m 25s | | trunk passed |
| +1 :green_heart: | shadedclient | 38m 49s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +1 :green_heart: | mvninstall | 0m 33s | | the patch passed |
| +1 :green_heart: | compile | 1m 21s | | the patch passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javac | 1m 21s | | the patch passed |
| +1 :green_heart: | compile | 1m 17s | | the patch passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | javac | 1m 17s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks
issues. |
| -0 :warning: | checkstyle | 0m 29s |
[/results-checkstyle-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6821/1/artifact/out/results-checkstyle-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-nodemanager.txt)
|
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager:
The patch generated 1 new + 1 unchanged - 0 fixed = 2 total (was 1) |
| +1 :green_heart: | mvnsite | 0m 34s | | the patch passed |
| +1 :green_heart: | javadoc | 0m 35s | | the patch passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javadoc | 0m 32s | | the patch passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | spotbugs | 1m 26s | | the patch passed |
| +1 :green_heart: | shadedclient | 38m 40s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart: | unit | 24m 55s | | hadoop-yarn-server-nodemanager
in the patch passed. |
| +1 :green_heart: | asflicense | 0m 34s | | The patch does not
generate ASF License warnings. |
| | | 190m 52s | | |
| Subsystem | Report/Notes |
|--:|:-|
| Docker | ClientAPI=1.44 ServerAPI=1.44 base:
https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6821/1/artifact/out/Dockerfile
|
| GITHUB PR | https://github.com/apache/hadoop/pull/6821 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall
mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | Linux e111bfa7f5ce 5.15.0-94-generic #104-Ubuntu SMP Tue Jan 9
15:25:40 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / d945e4e08428ca2ad25c524e55ce5afd990a055a |
| Default Java | Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| Multi-JDK versions |
/usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1
/usr/lib/jvm/java-8-openjdk-amd64:Private
Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| Test Results |
https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6821/1/testReport/ |
| Max. process+thread count
[jira] [Commented] (YARN-11692) Support mixed cgroup v1/v2 controller structure
[
https://issues.apache.org/jira/browse/YARN-11692?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17845916#comment-17845916
]
ASF GitHub Bot commented on YARN-11692:
---
brumi1024 commented on code in PR #6821:
URL: https://github.com/apache/hadoop/pull/6821#discussion_r1598441803
##
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java:
##
@@ -101,18 +120,18 @@ private static CGroupsHandler
getInitializedCGroupsHandler(Configuration conf)
*/
public static CGroupsHandler getCGroupsHandler() {
-return cGroupsHandler;
+return cgroupsV2Enabled ? cGroupsV2Handler : cGroupsV1Handler;
Review Comment:
getCGroupsHandler method is called from DefaultOOMHandler,
CGroupElasticMemoryController, which seemingly rely on v1 memory limits, so the
related changes should be addressed in
[YARN-11678](https://issues.apache.org/jira/browse/YARN-11678). It's also
called from the CGroupsResourceCalculator, which will handle the v1/v2
differences separately.
It's called from OCIContainerRuntime, but that class doesn't use it
cgroupHandler, DockerLinuxContainerRuntime calls it to check resource limits,
and RuncContainerRuntime calls it for checking and storing cpu limits. Having
this return the v1 or v2 handler based on the config will result in issues,
even in mixed mode, where for example the CPU is mounted in a v1 structure, so
I suggest leaving this as is for now, and updating it once
DockerLinuxContainerRuntime and RuncContainerRuntime is updated to handle the
v2 files.
##
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java:
##
@@ -63,35 +63,54 @@ public class ResourceHandlerModule {
* as resource metrics functionality. We need to ensure that the same
* instance is used for both.
*/
+ private static volatile CGroupsHandler cGroupsV1Handler;
+ private static volatile CGroupsHandler cGroupsV2Handler;
private static volatile TrafficControlBandwidthHandlerImpl
trafficControlBandwidthHandler;
private static volatile NetworkPacketTaggingHandlerImpl
networkPacketTaggingHandlerImpl;
- private static volatile CGroupsHandler cGroupsHandler;
private static volatile CGroupsBlkioResourceHandlerImpl
cGroupsBlkioResourceHandler;
private static volatile MemoryResourceHandler
cGroupsMemoryResourceHandler;
private static volatile CpuResourceHandler
cGroupsCpuResourceHandler;
- /**
- * Returns an initialized, thread-safe CGroupsHandler instance.
- */
- private static CGroupsHandler getInitializedCGroupsHandler(Configuration
conf)
+ private static void initializeCGroupsHandlers(Configuration conf) throws
ResourceHandlerException {
+initializeCGroupsV1Handler(conf);
+if (cgroupsV2Enabled) {
+ initializeCGroupsV2Handler(conf);
+}
+ }
+
+ private static void initializeCGroupsV1Handler(Configuration conf)
throws ResourceHandlerException {
-if (cGroupsHandler == null) {
+if (cGroupsV1Handler == null) {
synchronized (CGroupsHandler.class) {
-if (cGroupsHandler == null) {
- cGroupsHandler = cgroupsV2Enabled
- ? new CGroupsV2HandlerImpl(conf,
PrivilegedOperationExecutor.getInstance(conf))
- : new CGroupsHandlerImpl(conf,
PrivilegedOperationExecutor.getInstance(conf));
- LOG.debug("Value of CGroupsHandler is: {}", cGroupsHandler);
+if (cGroupsV1Handler == null) {
Review Comment:
Do we need the null check twice?
##
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml:
##
@@ -2087,6 +2087,20 @@
yarn.nodemanager.linux-container-executor.cgroups.mount-path
+
+This property sets the mount path for CGroups v2.
+ This parameter is optional, and needed to be set only in mixed mode,
+ when CGroups v2 is mounted in a different path than Cgroups v1.
+ For example, in hybrid mode, CGroups v1 controllers can be mounted in
+ /sys/fs/cgroup, while v2 can be mounted in /sys/fs/cgroup/unified folder.
Review Comment:
> v1 controllers can be mounted in /sys/fs/cgroup
Might be a bit misleading in this format, maybe say something like this:
"v1 controllers can be mounted under /sys/fs/cgroup/ (for example
/sys/fs/cgroup/cpu,cpuacct)". Otherwise it can seem like the cgroup v2 FS is
mounted under a cgroup v1 FS.
##
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java:
##
@@
[jira] [Commented] (YARN-11692) Support mixed cgroup v1/v2 controller structure
[ https://issues.apache.org/jira/browse/YARN-11692?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17845913#comment-17845913 ] ASF GitHub Bot commented on YARN-11692: --- hadoop-yetus commented on PR #6821: URL: https://github.com/apache/hadoop/pull/6821#issuecomment-2107523818 :broken_heart: **-1 overall** | Vote | Subsystem | Runtime | Logfile | Comment | |::|--:|:|::|:---:| | -1 :x: | patch | 0m 45s | | https://github.com/apache/hadoop/pull/6821 does not apply to trunk. Rebase required? Wrong Branch? See https://cwiki.apache.org/confluence/display/HADOOP/How+To+Contribute for help. | | Subsystem | Report/Notes | |--:|:-| | GITHUB PR | https://github.com/apache/hadoop/pull/6821 | | Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6821/2/console | | versions | git=2.45.0.windows.1 | | Powered by | Apache Yetus 0.14.0 https://yetus.apache.org | This message was automatically generated. > Support mixed cgroup v1/v2 controller structure > --- > > Key: YARN-11692 > URL: https://issues.apache.org/jira/browse/YARN-11692 > Project: Hadoop YARN > Issue Type: Sub-task >Reporter: Benjamin Teke >Assignee: Peter Szucs >Priority: Major > Labels: pull-request-available > > There were heavy changes on the device side in cgroup v2. To keep supporting > FGPAs and GPUs short term, mixed structures where some of the cgroup > controllers are from v1 while others from v2 should be supported. More info: > https://dropbear.xyz/2023/05/23/devices-with-cgroup-v2/ -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Reopened] (YARN-11669) [Umbrella] cgroup v2 support
[
https://issues.apache.org/jira/browse/YARN-11669?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Benjamin Teke reopened YARN-11669:
--
> [Umbrella] cgroup v2 support
>
>
> Key: YARN-11669
> URL: https://issues.apache.org/jira/browse/YARN-11669
> Project: Hadoop YARN
> Issue Type: New Feature
> Components: yarn
>Reporter: Ferenc Erdelyi
>Assignee: Benjamin Teke
>Priority: Major
> Fix For: 3.5.0
>
>
> cgroup v2 has some fundamental changes compared to v1. RHEL9, Ubuntu 22
> already moved to cgroup v2 as a default, hence YARN should support it. This
> umbrella tracks the required work.
> [Documentation|https://docs.kernel.org/admin-guide/cgroup-v2.html]
> A way to test the newly added features:
> # Turn on cgroup v1 based on the current
> [documentation|https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/NodeManagerCgroups.html].
> # System prerequisites:
> ## the file {{/etc/mtab}} should contain a mount path with the file system
> type {{cgroup2}}, by default this could be {{/sys/fs/cgroup}} on most OS's
> ## the {{cgroup.subtree_control}} file should contain the necessary
> controllers (update it with: {{echo "+cpu +io +memory" >
> cgroup.subtree_control}})
> ## either create the YARN hierarchy and give recursive access to the user
> running the NM on the node. The hierarchy is {{hadoop-yarn}} by default
> (controller by
> {{yarn.nodemanager.linux-container-executor.cgroups.hierarchy}}), and
> recursive mode is required, because as soon as the directory is created it
> will be filled with the controller files which YARN will try to edit.
> ### Alternatively if the NM process user has access rights on the
> {{/sys/fs/cgroup}} directory it'll try to create the hierarchy and update the
> {{cgroup.subtree_control}} file.
> # YARN configuration
> ## {{yarn.nodemanager.linux-container-executor.cgroups.mount-path}} should
> point to the directory where the cgroup2 structure is mounted and the
> {{hadoop-yarn}} hierarchy was created
> ## {{yarn.nodemanager.linux-container-executor.cgroups.v2.enabled}} should be
> set to {{true}}
> ## Enable a cgroup controller, like {{yarn. nodemanager. resource.
> cpu.enabled}}: {{true}}
> # Launch the NM and monitor the cgroup files on container launches (i.e:
> {{/sys/fs/cgroup/hadoop-yarn/container_id/cpu.weight}})
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-11669) [Umbrella] cgroup v2 support
[
https://issues.apache.org/jira/browse/YARN-11669?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Benjamin Teke updated YARN-11669:
-
Fix Version/s: (was: 3.5.0)
> [Umbrella] cgroup v2 support
>
>
> Key: YARN-11669
> URL: https://issues.apache.org/jira/browse/YARN-11669
> Project: Hadoop YARN
> Issue Type: New Feature
> Components: yarn
>Reporter: Ferenc Erdelyi
>Assignee: Benjamin Teke
>Priority: Major
>
> cgroup v2 has some fundamental changes compared to v1. RHEL9, Ubuntu 22
> already moved to cgroup v2 as a default, hence YARN should support it. This
> umbrella tracks the required work.
> [Documentation|https://docs.kernel.org/admin-guide/cgroup-v2.html]
> A way to test the newly added features:
> # Turn on cgroup v1 based on the current
> [documentation|https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/NodeManagerCgroups.html].
> # System prerequisites:
> ## the file {{/etc/mtab}} should contain a mount path with the file system
> type {{cgroup2}}, by default this could be {{/sys/fs/cgroup}} on most OS's
> ## the {{cgroup.subtree_control}} file should contain the necessary
> controllers (update it with: {{echo "+cpu +io +memory" >
> cgroup.subtree_control}})
> ## either create the YARN hierarchy and give recursive access to the user
> running the NM on the node. The hierarchy is {{hadoop-yarn}} by default
> (controller by
> {{yarn.nodemanager.linux-container-executor.cgroups.hierarchy}}), and
> recursive mode is required, because as soon as the directory is created it
> will be filled with the controller files which YARN will try to edit.
> ### Alternatively if the NM process user has access rights on the
> {{/sys/fs/cgroup}} directory it'll try to create the hierarchy and update the
> {{cgroup.subtree_control}} file.
> # YARN configuration
> ## {{yarn.nodemanager.linux-container-executor.cgroups.mount-path}} should
> point to the directory where the cgroup2 structure is mounted and the
> {{hadoop-yarn}} hierarchy was created
> ## {{yarn.nodemanager.linux-container-executor.cgroups.v2.enabled}} should be
> set to {{true}}
> ## Enable a cgroup controller, like {{yarn. nodemanager. resource.
> cpu.enabled}}: {{true}}
> # Launch the NM and monitor the cgroup files on container launches (i.e:
> {{/sys/fs/cgroup/hadoop-yarn/container_id/cpu.weight}})
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11689) Update the cgroup v2 init error handling to provide more straightforward error messages
[ https://issues.apache.org/jira/browse/YARN-11689?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17845877#comment-17845877 ] ASF GitHub Bot commented on YARN-11689: --- brumi1024 merged PR #6810: URL: https://github.com/apache/hadoop/pull/6810 > Update the cgroup v2 init error handling to provide more straightforward > error messages > --- > > Key: YARN-11689 > URL: https://issues.apache.org/jira/browse/YARN-11689 > Project: Hadoop YARN > Issue Type: Sub-task >Reporter: Benjamin Teke >Assignee: Benjamin Teke >Priority: Major > Labels: pull-request-available > > The method AbstractCGroupsHandler.getErrorWithDetails hides quite a lot of > information. The cgroup v2 init should be more stable and it should be > updated to show the exact step where the it failed. -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Resolved] (YARN-11669) [Umbrella] cgroup v2 support
[
https://issues.apache.org/jira/browse/YARN-11669?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Benjamin Teke resolved YARN-11669.
--
Fix Version/s: 3.5.0
Hadoop Flags: Reviewed
Resolution: Fixed
> [Umbrella] cgroup v2 support
>
>
> Key: YARN-11669
> URL: https://issues.apache.org/jira/browse/YARN-11669
> Project: Hadoop YARN
> Issue Type: New Feature
> Components: yarn
>Reporter: Ferenc Erdelyi
>Assignee: Benjamin Teke
>Priority: Major
> Fix For: 3.5.0
>
>
> cgroup v2 has some fundamental changes compared to v1. RHEL9, Ubuntu 22
> already moved to cgroup v2 as a default, hence YARN should support it. This
> umbrella tracks the required work.
> [Documentation|https://docs.kernel.org/admin-guide/cgroup-v2.html]
> A way to test the newly added features:
> # Turn on cgroup v1 based on the current
> [documentation|https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/NodeManagerCgroups.html].
> # System prerequisites:
> ## the file {{/etc/mtab}} should contain a mount path with the file system
> type {{cgroup2}}, by default this could be {{/sys/fs/cgroup}} on most OS's
> ## the {{cgroup.subtree_control}} file should contain the necessary
> controllers (update it with: {{echo "+cpu +io +memory" >
> cgroup.subtree_control}})
> ## either create the YARN hierarchy and give recursive access to the user
> running the NM on the node. The hierarchy is {{hadoop-yarn}} by default
> (controller by
> {{yarn.nodemanager.linux-container-executor.cgroups.hierarchy}}), and
> recursive mode is required, because as soon as the directory is created it
> will be filled with the controller files which YARN will try to edit.
> ### Alternatively if the NM process user has access rights on the
> {{/sys/fs/cgroup}} directory it'll try to create the hierarchy and update the
> {{cgroup.subtree_control}} file.
> # YARN configuration
> ## {{yarn.nodemanager.linux-container-executor.cgroups.mount-path}} should
> point to the directory where the cgroup2 structure is mounted and the
> {{hadoop-yarn}} hierarchy was created
> ## {{yarn.nodemanager.linux-container-executor.cgroups.v2.enabled}} should be
> set to {{true}}
> ## Enable a cgroup controller, like {{yarn. nodemanager. resource.
> cpu.enabled}}: {{true}}
> # Launch the NM and monitor the cgroup files on container launches (i.e:
> {{/sys/fs/cgroup/hadoop-yarn/container_id/cpu.weight}})
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11689) Update the cgroup v2 init error handling to provide more straightforward error messages
[ https://issues.apache.org/jira/browse/YARN-11689?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17845876#comment-17845876 ] ASF GitHub Bot commented on YARN-11689: --- brumi1024 commented on PR #6810: URL: https://github.com/apache/hadoop/pull/6810#issuecomment-2107265917 Thanks @K0K0V0K and @p-szucs for the review, merging to trunk. > Update the cgroup v2 init error handling to provide more straightforward > error messages > --- > > Key: YARN-11689 > URL: https://issues.apache.org/jira/browse/YARN-11689 > Project: Hadoop YARN > Issue Type: Sub-task >Reporter: Benjamin Teke >Assignee: Benjamin Teke >Priority: Major > Labels: pull-request-available > > The method AbstractCGroupsHandler.getErrorWithDetails hides quite a lot of > information. The cgroup v2 init should be more stable and it should be > updated to show the exact step where the it failed. -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11692) Support mixed cgroup v1/v2 controller structure
[ https://issues.apache.org/jira/browse/YARN-11692?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17845864#comment-17845864 ] ASF GitHub Bot commented on YARN-11692: --- p-szucs opened a new pull request, #6821: URL: https://github.com/apache/hadoop/pull/6821 Change-Id: I10a024b3fa88aaeb5dfcd9173f53200b00ccc7df ### Description of PR There were heavy changes on the device side in cgroup v2. To keep supporting FGPAs and GPUs short term, mixed structures where some of the cgroup controllers are from v1 while others from v2 should be supported. More info: https://dropbear.xyz/2023/05/23/devices-with-cgroup-v2/ ### How was this patch tested? ### For code changes: - [x] Does the title or this PR starts with the corresponding JIRA issue id (e.g. 'HADOOP-17799. Your PR title ...')? - [ ] Object storage: have the integration tests been executed and the endpoint declared according to the connector-specific documentation? - [ ] If adding new dependencies to the code, are these dependencies licensed in a way that is compatible for inclusion under [ASF 2.0](http://www.apache.org/legal/resolved.html#category-a)? - [ ] If applicable, have you updated the `LICENSE`, `LICENSE-binary`, `NOTICE-binary` files? > Support mixed cgroup v1/v2 controller structure > --- > > Key: YARN-11692 > URL: https://issues.apache.org/jira/browse/YARN-11692 > Project: Hadoop YARN > Issue Type: Sub-task >Reporter: Benjamin Teke >Assignee: Peter Szucs >Priority: Major > > There were heavy changes on the device side in cgroup v2. To keep supporting > FGPAs and GPUs short term, mixed structures where some of the cgroup > controllers are from v1 while others from v2 should be supported. More info: > https://dropbear.xyz/2023/05/23/devices-with-cgroup-v2/ -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-11692) Support mixed cgroup v1/v2 controller structure
[ https://issues.apache.org/jira/browse/YARN-11692?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] ASF GitHub Bot updated YARN-11692: -- Labels: pull-request-available (was: ) > Support mixed cgroup v1/v2 controller structure > --- > > Key: YARN-11692 > URL: https://issues.apache.org/jira/browse/YARN-11692 > Project: Hadoop YARN > Issue Type: Sub-task >Reporter: Benjamin Teke >Assignee: Peter Szucs >Priority: Major > Labels: pull-request-available > > There were heavy changes on the device side in cgroup v2. To keep supporting > FGPAs and GPUs short term, mixed structures where some of the cgroup > controllers are from v1 while others from v2 should be supported. More info: > https://dropbear.xyz/2023/05/23/devices-with-cgroup-v2/ -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-11696) Add debug-level logs in RMAppImpl
[
https://issues.apache.org/jira/browse/YARN-11696?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Susheel Gupta updated YARN-11696:
-
Summary: Add debug-level logs in RMAppImpl (was: Add debug-level logs in
RMAppImpl#aggregateLogReport and RMAppImpl#getLogAggregationStatusForAppReport)
> Add debug-level logs in RMAppImpl
> -
>
> Key: YARN-11696
> URL: https://issues.apache.org/jira/browse/YARN-11696
> Project: Hadoop YARN
> Issue Type: Improvement
> Components: yarn
>Reporter: Susheel Gupta
>Assignee: Susheel Gupta
>Priority: Minor
>
> The events keep increasing in event-queue and many event thread are blocked.
> To discover the deadlocking threads, add a few debug level logs to
> RMAppImpl#aggregateLogReport and
> RMAppImpl#getLogAggregationStatusForAppReport.
> {code:java}
> "RM Event dispatcher" #93 prio=5 os_prio=0 tid=0x7fcb67120800 nid=0x13e62
> waiting on condition [0x7fbef632a000]
> java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for <0x7fc44cada248> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1199)
> at
> java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(ReentrantReadWriteLock.java:943)
> at
> org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl.aggregateLogReport(RMAppImpl.java:1799)
> at
> org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl.handleLogAggregationStatus(RMNodeImpl.java:1478)
> at
> org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl.access$500(RMNodeImpl.java:104)
> at
> org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl$StatusUpdateWhenHealthyTransition.transition(RMNodeImpl.java:1239)
> at
> org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl$StatusUpdateWhenHealthyTransition.transition(RMNodeImpl.java:1195)
> at
> org.apache.hadoop.yarn.state.StateMachineFactory$MultipleInternalArc.doTransition(StateMachineFactory.java:385)
> at
> org.apache.hadoop.yarn.state.StateMachineFactory.doTransition(StateMachineFactory.java:302)
> at
> org.apache.hadoop.yarn.state.StateMachineFactory.access$500(StateMachineFactory.java:46)
> at
> org.apache.hadoop.yarn.state.StateMachineFactory$InternalStateMachine.doTransition(StateMachineFactory.java:487)
> - locked <0x7fc04c0b6970> (a
> org.apache.hadoop.yarn.state.StateMachineFactory$InternalStateMachine)
> at
> org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl.handle(RMNodeImpl.java:667)
> at
> org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl.handle(RMNodeImpl.java:101)
> at
> org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$NodeEventDispatcher.handle(ResourceManager.java:1124)
> at
> org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$NodeEventDispatcher.handle(ResourceManager.java:1108)
> at
> org.apache.hadoop.yarn.event.AsyncDispatcher.dispatch(AsyncDispatcher.java:219)
> at
> org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:133)
> at java.lang.Thread.run(Thread.java:748) {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Created] (YARN-11696) Add debug-level logs in RMAppImpl#aggregateLogReport and RMAppImpl#getLogAggregationStatusForAppReport
Susheel Gupta created YARN-11696:
Summary: Add debug-level logs in RMAppImpl#aggregateLogReport and
RMAppImpl#getLogAggregationStatusForAppReport
Key: YARN-11696
URL: https://issues.apache.org/jira/browse/YARN-11696
Project: Hadoop YARN
Issue Type: Improvement
Components: yarn
Reporter: Susheel Gupta
Assignee: Susheel Gupta
The events keep increasing in event-queue and many event thread are blocked.
To discover the deadlocking threads, add a few debug level logs to
RMAppImpl#aggregateLogReport and RMAppImpl#getLogAggregationStatusForAppReport.
{code:java}
"RM Event dispatcher" #93 prio=5 os_prio=0 tid=0x7fcb67120800 nid=0x13e62
waiting on condition [0x7fbef632a000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x7fc44cada248> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1199)
at
java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(ReentrantReadWriteLock.java:943)
at
org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl.aggregateLogReport(RMAppImpl.java:1799)
at
org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl.handleLogAggregationStatus(RMNodeImpl.java:1478)
at
org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl.access$500(RMNodeImpl.java:104)
at
org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl$StatusUpdateWhenHealthyTransition.transition(RMNodeImpl.java:1239)
at
org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl$StatusUpdateWhenHealthyTransition.transition(RMNodeImpl.java:1195)
at
org.apache.hadoop.yarn.state.StateMachineFactory$MultipleInternalArc.doTransition(StateMachineFactory.java:385)
at
org.apache.hadoop.yarn.state.StateMachineFactory.doTransition(StateMachineFactory.java:302)
at
org.apache.hadoop.yarn.state.StateMachineFactory.access$500(StateMachineFactory.java:46)
at
org.apache.hadoop.yarn.state.StateMachineFactory$InternalStateMachine.doTransition(StateMachineFactory.java:487)
- locked <0x7fc04c0b6970> (a
org.apache.hadoop.yarn.state.StateMachineFactory$InternalStateMachine)
at
org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl.handle(RMNodeImpl.java:667)
at
org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl.handle(RMNodeImpl.java:101)
at
org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$NodeEventDispatcher.handle(ResourceManager.java:1124)
at
org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$NodeEventDispatcher.handle(ResourceManager.java:1108)
at
org.apache.hadoop.yarn.event.AsyncDispatcher.dispatch(AsyncDispatcher.java:219)
at
org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:133)
at java.lang.Thread.run(Thread.java:748) {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11689) Update the cgroup v2 init error handling to provide more straightforward error messages
[ https://issues.apache.org/jira/browse/YARN-11689?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17845818#comment-17845818 ] ASF GitHub Bot commented on YARN-11689: --- K0K0V0K commented on PR #6810: URL: https://github.com/apache/hadoop/pull/6810#issuecomment-2106894049 Thanks @brumi1024 for the fix! LGTM. > Update the cgroup v2 init error handling to provide more straightforward > error messages > --- > > Key: YARN-11689 > URL: https://issues.apache.org/jira/browse/YARN-11689 > Project: Hadoop YARN > Issue Type: Sub-task >Reporter: Benjamin Teke >Assignee: Benjamin Teke >Priority: Major > Labels: pull-request-available > > The method AbstractCGroupsHandler.getErrorWithDetails hides quite a lot of > information. The cgroup v2 init should be more stable and it should be > updated to show the exact step where the it failed. -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11471) FederationStateStoreFacade Cache Support Caffeine
[ https://issues.apache.org/jira/browse/YARN-11471?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17845780#comment-17845780 ] ASF GitHub Bot commented on YARN-11471: --- slfan1989 commented on code in PR #6795: URL: https://github.com/apache/hadoop/pull/6795#discussion_r1597873381 ## hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md: ## @@ -536,7 +536,7 @@ To enable cross-origin support (CORS) for the Yarn Router, please set the follow How to configure Router Cache Cache is enabled by default. When we set the `yarn.federation.cache-ttl.secs` parameter and its value is greater than 0, Cache will be enabled. -We currently provide two Cache implementations: `JCache` and `GuavaCache`. +We currently provide Three Cache implementations: `JCache` , `GuavaCache`, `CaffeineCache` Review Comment: Thanks for your suggestions! I will improve the documentation. > FederationStateStoreFacade Cache Support Caffeine > - > > Key: YARN-11471 > URL: https://issues.apache.org/jira/browse/YARN-11471 > Project: Hadoop YARN > Issue Type: Improvement > Components: federation >Affects Versions: 3.4.0 >Reporter: Shilun Fan >Assignee: Shilun Fan >Priority: Major > Labels: pull-request-available > > FederationStateStoreFacade Cache Support Caffeine -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11471) FederationStateStoreFacade Cache Support Caffeine
[ https://issues.apache.org/jira/browse/YARN-11471?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17845729#comment-17845729 ] ASF GitHub Bot commented on YARN-11471: --- goiri commented on code in PR #6795: URL: https://github.com/apache/hadoop/pull/6795#discussion_r1597672074 ## hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md: ## @@ -536,7 +536,7 @@ To enable cross-origin support (CORS) for the Yarn Router, please set the follow How to configure Router Cache Cache is enabled by default. When we set the `yarn.federation.cache-ttl.secs` parameter and its value is greater than 0, Cache will be enabled. -We currently provide two Cache implementations: `JCache` and `GuavaCache`. +We currently provide Three Cache implementations: `JCache` , `GuavaCache`, `CaffeineCache` Review Comment: `three` with lower case. > FederationStateStoreFacade Cache Support Caffeine > - > > Key: YARN-11471 > URL: https://issues.apache.org/jira/browse/YARN-11471 > Project: Hadoop YARN > Issue Type: Improvement > Components: federation >Affects Versions: 3.4.0 >Reporter: Shilun Fan >Assignee: Shilun Fan >Priority: Major > Labels: pull-request-available > > FederationStateStoreFacade Cache Support Caffeine -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11471) FederationStateStoreFacade Cache Support Caffeine
[
https://issues.apache.org/jira/browse/YARN-11471?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17845620#comment-17845620
]
ASF GitHub Bot commented on YARN-11471:
---
hadoop-yetus commented on PR #6795:
URL: https://github.com/apache/hadoop/pull/6795#issuecomment-2105893097
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 01s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 01s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 01s | | detect-secrets was not available.
|
| +0 :ok: | shellcheck | 0m 02s | | Shellcheck was not available. |
| +0 :ok: | shelldocs | 0m 02s | | Shelldocs was not available. |
| +0 :ok: | xmllint | 0m 00s | | xmllint was not available. |
| +0 :ok: | spotbugs | 0m 00s | | spotbugs executables are not
available. |
| +0 :ok: | markdownlint | 0m 00s | | markdownlint was not available.
|
| +1 :green_heart: | @author | 0m 00s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 01s | | The patch appears to
include 1 new or modified test files. |
_ trunk Compile Tests _ |
| +0 :ok: | mvndep | 2m 16s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 90m 30s | | trunk passed |
| +1 :green_heart: | compile | 40m 45s | | trunk passed |
| +1 :green_heart: | checkstyle | 6m 00s | | trunk passed |
| -1 :x: | mvnsite | 23m 14s |
[/branch-mvnsite-root.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6795/4/artifact/out/branch-mvnsite-root.txt)
| root in trunk failed. |
| +1 :green_heart: | javadoc | 18m 33s | | trunk passed |
| +1 :green_heart: | shadedclient | 198m 44s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +0 :ok: | mvndep | 2m 26s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 98m 44s | | the patch passed |
| +1 :green_heart: | compile | 40m 42s | | the patch passed |
| +1 :green_heart: | javac | 40m 42s | | the patch passed |
| +1 :green_heart: | blanks | 0m 00s | | The patch has no blanks
issues. |
| +1 :green_heart: | checkstyle | 6m 04s | | the patch passed |
| -1 :x: | mvnsite | 24m 17s |
[/patch-mvnsite-root.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6795/4/artifact/out/patch-mvnsite-root.txt)
| root in the patch failed. |
| +1 :green_heart: | javadoc | 17m 08s | | the patch passed |
| +1 :green_heart: | shadedclient | 210m 44s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart: | asflicense | 6m 30s | | The patch does not
generate ASF License warnings. |
| | | 697m 52s | | |
| Subsystem | Report/Notes |
|--:|:-|
| GITHUB PR | https://github.com/apache/hadoop/pull/6795 |
| Optional Tests | dupname asflicense codespell detsecrets shellcheck
shelldocs compile javac javadoc mvninstall mvnsite unit shadedclient xmllint
spotbugs checkstyle markdownlint |
| uname | MINGW64_NT-10.0-17763 6f5c5539c887 3.4.10-87d57229.x86_64
2024-02-14 20:17 UTC x86_64 Msys |
| Build tool | maven |
| Personality | /c/hadoop/dev-support/bin/hadoop.sh |
| git revision | trunk / 8c86187d7699aa8f2610a96e3c026e13b58fffc9 |
| Default Java | Azul Systems, Inc.-1.8.0_332-b09 |
| Test Results |
https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6795/4/testReport/
|
| modules | C: hadoop-project
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site . U: . |
| Console output |
https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6795/4/console
|
| versions | git=2.44.0.windows.1 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
> FederationStateStoreFacade Cache Support Caffeine
> -
>
> Key: YARN-11471
> URL: https://issues.apache.org/jira/browse/YARN-11471
> Project: Hadoop YARN
> Issue Type: Improvement
> Components: federation
>Affects Versions: 3.4.0
>Reporter: Shilun Fan
>Assignee: Shilun Fan
>P
[jira] [Commented] (YARN-11010) YARN ui2 hangs on the Queues page when the scheduler response contains NaN values
[
https://issues.apache.org/jira/browse/YARN-11010?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17845558#comment-17845558
]
Xie YiFan commented on YARN-11010:
--
[~tdomok] [~bkosztolnik] hi, I try to reproduce this issue in trunk branch
and find that it already unable to use percentage mode on leaf queue when set
parent in absolute mode. But i can reproduce this issue in another easy way.
Reproduction steps:
# config DominantResourceCalculator
# set one child queue with [memory=0,vcores=0] in absolute mode
The root cause is divide zero by zero which return NaN as a result. YARN-9019
fixed NaN/Infinity issue with ratio function in DefaultResourceCalculator and
DominantResourceCalculator. The DefaultResourceCalculator.divide is implemented
by ratio function but DominantResourceCalculator.divide is not.
So DominantResourceCalculator.divide function may return Nan result.
> YARN ui2 hangs on the Queues page when the scheduler response contains NaN
> values
> -
>
> Key: YARN-11010
> URL: https://issues.apache.org/jira/browse/YARN-11010
> Project: Hadoop YARN
> Issue Type: Bug
> Components: yarn-ui-v2
>Affects Versions: 3.4.0
>Reporter: Tamas Domok
>Assignee: Xie YiFan
>Priority: Major
> Attachments: capacity-scheduler.xml, shresponse.json
>
>
> When the scheduler response contains NaN values for capacity and maxCapacity
> the UI2 hangs on the Queues page. The console log shows the following error:
> {code:java}
> SyntaxError: Unexpected token N in JSON at position 666 {code}
> The scheduler response:
> {code:java}
> "maxCapacity": NaN,
> "absoluteMaxCapacity": NaN, {code}
> NaN, infinity, -infinity is not valid in JSON syntax:
> https://www.json.org/json-en.html
> This might be related as well: YARN-10452
>
> I managed to reproduce this with AQCv1, where I set the parent queue's
> capacity in absolute mode, then I used percentage mode on the
> leaf-queue-template. I'm not sure if this is a valid configuration, however
> there is no error or warning in RM logs about any configuration error. To
> trigger the issue the DominantResourceCalculator must be used. (When using
> absolute mode on the leaf-queue-template this issue is not re-producible,
> further details on: YARN-10922).
>
> Reproduction steps:
> # Start the cluster with the attached configuration
> # Check the Queues page on UI2 (it should work at this point)
> # Send an example job (yarn jar hadoop-mapreduce-examples-3.4.0-SNAPSHOT.jar
> pi 1 10)
> # Check the Queues page on UI2 (it should not be working at this point)
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11689) Update the cgroup v2 init error handling to provide more straightforward error messages
[
https://issues.apache.org/jira/browse/YARN-11689?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17845534#comment-17845534
]
ASF GitHub Bot commented on YARN-11689:
---
hadoop-yetus commented on PR #6810:
URL: https://github.com/apache/hadoop/pull/6810#issuecomment-2105549930
:confetti_ball: **+1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 00s | | No case conflicting files
found. |
| +0 :ok: | spotbugs | 0m 01s | | spotbugs executables are not
available. |
| +0 :ok: | codespell | 0m 01s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 01s | | detect-secrets was not available.
|
| +1 :green_heart: | @author | 0m 00s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 00s | | The patch appears to
include 1 new or modified test files. |
_ trunk Compile Tests _ |
| +1 :green_heart: | mvninstall | 93m 34s | | trunk passed |
| +1 :green_heart: | compile | 5m 30s | | trunk passed |
| +1 :green_heart: | checkstyle | 4m 51s | | trunk passed |
| +1 :green_heart: | mvnsite | 5m 30s | | trunk passed |
| +1 :green_heart: | javadoc | 5m 16s | | trunk passed |
| +1 :green_heart: | shadedclient | 151m 20s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +1 :green_heart: | mvninstall | 3m 09s | | the patch passed |
| +1 :green_heart: | compile | 2m 44s | | the patch passed |
| +1 :green_heart: | javac | 2m 44s | | the patch passed |
| +1 :green_heart: | blanks | 0m 00s | | The patch has no blanks
issues. |
| +1 :green_heart: | checkstyle | 2m 15s | | the patch passed |
| +1 :green_heart: | mvnsite | 2m 57s | | the patch passed |
| +1 :green_heart: | javadoc | 2m 43s | | the patch passed |
| +1 :green_heart: | shadedclient | 163m 42s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart: | asflicense | 5m 49s | | The patch does not
generate ASF License warnings. |
| | | 433m 55s | | |
| Subsystem | Report/Notes |
|--:|:-|
| GITHUB PR | https://github.com/apache/hadoop/pull/6810 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall
mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | MINGW64_NT-10.0-17763 0d202dd2dc0f 3.4.10-87d57229.x86_64
2024-02-14 20:17 UTC x86_64 Msys |
| Build tool | maven |
| Personality | /c/hadoop/dev-support/bin/hadoop.sh |
| git revision | trunk / 8f96d13d9ccb426da32cd5df4004f7203fdce752 |
| Default Java | Azul Systems, Inc.-1.8.0_332-b09 |
| Test Results |
https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6810/1/testReport/
|
| modules | C:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager
U:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager
|
| Console output |
https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6810/1/console
|
| versions | git=2.45.0.windows.1 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
> Update the cgroup v2 init error handling to provide more straightforward
> error messages
> ---
>
> Key: YARN-11689
> URL: https://issues.apache.org/jira/browse/YARN-11689
> Project: Hadoop YARN
> Issue Type: Sub-task
>Reporter: Benjamin Teke
>Assignee: Benjamin Teke
>Priority: Major
> Labels: pull-request-available
>
> The method AbstractCGroupsHandler.getErrorWithDetails hides quite a lot of
> information. The cgroup v2 init should be more stable and it should be
> updated to show the exact step where the it failed.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11471) FederationStateStoreFacade Cache Support Caffeine
[
https://issues.apache.org/jira/browse/YARN-11471?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17845513#comment-17845513
]
ASF GitHub Bot commented on YARN-11471:
---
hadoop-yetus commented on PR #6795:
URL: https://github.com/apache/hadoop/pull/6795#issuecomment-2105303552
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 01s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 02s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 02s | | detect-secrets was not available.
|
| +0 :ok: | shellcheck | 0m 02s | | Shellcheck was not available. |
| +0 :ok: | shelldocs | 0m 02s | | Shelldocs was not available. |
| +0 :ok: | xmllint | 0m 00s | | xmllint was not available. |
| +0 :ok: | spotbugs | 0m 00s | | spotbugs executables are not
available. |
| +0 :ok: | markdownlint | 0m 00s | | markdownlint was not available.
|
| +1 :green_heart: | @author | 0m 01s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 00s | | The patch appears to
include 1 new or modified test files. |
_ trunk Compile Tests _ |
| +0 :ok: | mvndep | 2m 21s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 94m 17s | | trunk passed |
| +1 :green_heart: | compile | 42m 36s | | trunk passed |
| +1 :green_heart: | checkstyle | 6m 22s | | trunk passed |
| -1 :x: | mvnsite | 25m 38s |
[/branch-mvnsite-root.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6795/3/artifact/out/branch-mvnsite-root.txt)
| root in trunk failed. |
| +1 :green_heart: | javadoc | 18m 48s | | trunk passed |
| +1 :green_heart: | shadedclient | 210m 12s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +0 :ok: | mvndep | 2m 38s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 99m 57s | | the patch passed |
| +1 :green_heart: | compile | 43m 37s | | the patch passed |
| +1 :green_heart: | javac | 43m 37s | | the patch passed |
| +1 :green_heart: | blanks | 0m 00s | | The patch has no blanks
issues. |
| +1 :green_heart: | checkstyle | 7m 33s | | the patch passed |
| -1 :x: | mvnsite | 24m 57s |
[/patch-mvnsite-root.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6795/3/artifact/out/patch-mvnsite-root.txt)
| root in the patch failed. |
| +1 :green_heart: | javadoc | 17m 06s | | the patch passed |
| +1 :green_heart: | shadedclient | 217m 51s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart: | asflicense | 7m 45s | | The patch does not
generate ASF License warnings. |
| | | 727m 24s | | |
| Subsystem | Report/Notes |
|--:|:-|
| GITHUB PR | https://github.com/apache/hadoop/pull/6795 |
| Optional Tests | dupname asflicense codespell detsecrets shellcheck
shelldocs compile javac javadoc mvninstall mvnsite unit shadedclient xmllint
spotbugs checkstyle markdownlint |
| uname | MINGW64_NT-10.0-17763 7d8350823e1a 3.4.10-87d57229.x86_64
2024-02-14 20:17 UTC x86_64 Msys |
| Build tool | maven |
| Personality | /c/hadoop/dev-support/bin/hadoop.sh |
| git revision | trunk / ac7dbd1c1c8ff4640ae2a295ba84f6b68a6b810e |
| Default Java | Azul Systems, Inc.-1.8.0_332-b09 |
| Test Results |
https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6795/3/testReport/
|
| modules | C: hadoop-project
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site . U: . |
| Console output |
https://ci-hadoop.apache.org/job/hadoop-multibranch-windows-10/job/PR-6795/3/console
|
| versions | git=2.45.0.windows.1 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
> FederationStateStoreFacade Cache Support Caffeine
> -
>
> Key: YARN-11471
> URL: https://issues.apache.org/jira/browse/YARN-11471
> Project: Hadoop YARN
> Issue Type: Improvement
> Components: federation
>Affects Versions: 3.4.0
>Reporter: Shilun Fan
>Assignee: Shilun Fan
>Priority: Major
> Labels: pull-request-available
>
> FederationStateStoreFacade Cache Support Caffeine
--
This messag
[jira] [Commented] (YARN-11689) Update the cgroup v2 init error handling to provide more straightforward error messages
[
https://issues.apache.org/jira/browse/YARN-11689?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17845469#comment-17845469
]
ASF GitHub Bot commented on YARN-11689:
---
hadoop-yetus commented on PR #6810:
URL: https://github.com/apache/hadoop/pull/6810#issuecomment-2105173101
:confetti_ball: **+1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 11m 53s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 1s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 1s | | detect-secrets was not available.
|
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to
include 1 new or modified test files. |
_ trunk Compile Tests _ |
| +1 :green_heart: | mvninstall | 44m 11s | | trunk passed |
| +1 :green_heart: | compile | 1m 24s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | compile | 1m 23s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | checkstyle | 0m 37s | | trunk passed |
| +1 :green_heart: | mvnsite | 0m 44s | | trunk passed |
| +1 :green_heart: | javadoc | 0m 46s | | trunk passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javadoc | 0m 39s | | trunk passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | spotbugs | 1m 25s | | trunk passed |
| +1 :green_heart: | shadedclient | 33m 23s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +1 :green_heart: | mvninstall | 0m 33s | | the patch passed |
| +1 :green_heart: | compile | 1m 17s | | the patch passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javac | 1m 17s | | the patch passed |
| +1 :green_heart: | compile | 1m 14s | | the patch passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | javac | 1m 14s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks
issues. |
| +1 :green_heart: | checkstyle | 0m 27s | | the patch passed |
| +1 :green_heart: | mvnsite | 0m 36s | | the patch passed |
| +1 :green_heart: | javadoc | 0m 33s | | the patch passed with JDK
Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1 |
| +1 :green_heart: | javadoc | 0m 31s | | the patch passed with JDK
Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| +1 :green_heart: | spotbugs | 1m 23s | | the patch passed |
| +1 :green_heart: | shadedclient | 33m 10s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart: | unit | 25m 8s | | hadoop-yarn-server-nodemanager
in the patch passed. |
| +1 :green_heart: | asflicense | 0m 36s | | The patch does not
generate ASF License warnings. |
| | | 166m 8s | | |
| Subsystem | Report/Notes |
|--:|:-|
| Docker | ClientAPI=1.44 ServerAPI=1.44 base:
https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6810/1/artifact/out/Dockerfile
|
| GITHUB PR | https://github.com/apache/hadoop/pull/6810 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall
mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | Linux 2c9928885b0c 5.15.0-94-generic #104-Ubuntu SMP Tue Jan 9
15:25:40 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / 8f96d13d9ccb426da32cd5df4004f7203fdce752 |
| Default Java | Private Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| Multi-JDK versions |
/usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.22+7-post-Ubuntu-0ubuntu220.04.1
/usr/lib/jvm/java-8-openjdk-amd64:Private
Build-1.8.0_402-8u402-ga-2ubuntu1~20.04-b06 |
| Test Results |
https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6810/1/testReport/ |
| Max. process+thread count | 552 (vs. ulimit of 5500) |
| modules | C:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager
U:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager
|
| Console output |
https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-6810/1/console |
| versions | git=2.25.1 maven=3.6.3 spotbugs=4.2.2 |
| Powered by | Apache Yetus 0.14.0 https
[jira] [Updated] (YARN-11689) Update the cgroup v2 init error handling to provide more straightforward error messages
[ https://issues.apache.org/jira/browse/YARN-11689?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] ASF GitHub Bot updated YARN-11689: -- Labels: pull-request-available (was: ) > Update the cgroup v2 init error handling to provide more straightforward > error messages > --- > > Key: YARN-11689 > URL: https://issues.apache.org/jira/browse/YARN-11689 > Project: Hadoop YARN > Issue Type: Sub-task >Reporter: Benjamin Teke >Assignee: Benjamin Teke >Priority: Major > Labels: pull-request-available > > The method AbstractCGroupsHandler.getErrorWithDetails hides quite a lot of > information. The cgroup v2 init should be more stable and it should be > updated to show the exact step where the it failed. -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11689) Update the cgroup v2 init error handling to provide more straightforward error messages
[ https://issues.apache.org/jira/browse/YARN-11689?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17845427#comment-17845427 ] ASF GitHub Bot commented on YARN-11689: --- brumi1024 opened a new pull request, #6810: URL: https://github.com/apache/hadoop/pull/6810 ### Description of PR The cgroup v2 init error handling was updated to show the concrete step it failed, and also a bug was fixed that disallowed the automatic creation of the YARN cgroup hierarchy. ### How was this patch tested? Unit test, and deployed it on a cluster with cgroup v2 enabled NMs. ### For code changes: - [ ] Does the title or this PR starts with the corresponding JIRA issue id (e.g. 'HADOOP-17799. Your PR title ...')? - [ ] Object storage: have the integration tests been executed and the endpoint declared according to the connector-specific documentation? - [ ] If adding new dependencies to the code, are these dependencies licensed in a way that is compatible for inclusion under [ASF 2.0](http://www.apache.org/legal/resolved.html#category-a)? - [ ] If applicable, have you updated the `LICENSE`, `LICENSE-binary`, `NOTICE-binary` files? > Update the cgroup v2 init error handling to provide more straightforward > error messages > --- > > Key: YARN-11689 > URL: https://issues.apache.org/jira/browse/YARN-11689 > Project: Hadoop YARN > Issue Type: Sub-task >Reporter: Benjamin Teke >Assignee: Benjamin Teke >Priority: Major > > The method AbstractCGroupsHandler.getErrorWithDetails hides quite a lot of > information. The cgroup v2 init should be more stable and it should be > updated to show the exact step where the it failed. -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-11689) Update the cgroup v2 init error handling to provide more straightforward error messages
[ https://issues.apache.org/jira/browse/YARN-11689?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Benjamin Teke updated YARN-11689: - Description: The method AbstractCGroupsHandler.getErrorWithDetails hides quite a lot of information. The cgroup v2 init should be more stable and it should be updated to show the exact step where the it failed. (was: The method AbstractCGroupsHandler.getErrorWithDetails hides quite a lot of information. It would be useful to show the underlying exception and it's message as well, by default.) > Update the cgroup v2 init error handling to provide more straightforward > error messages > --- > > Key: YARN-11689 > URL: https://issues.apache.org/jira/browse/YARN-11689 > Project: Hadoop YARN > Issue Type: Sub-task >Reporter: Benjamin Teke >Assignee: Benjamin Teke >Priority: Major > > The method AbstractCGroupsHandler.getErrorWithDetails hides quite a lot of > information. The cgroup v2 init should be more stable and it should be > updated to show the exact step where the it failed. -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-11689) Update the cgroup v2 init error handling to provide more straightforward error messages
[ https://issues.apache.org/jira/browse/YARN-11689?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Benjamin Teke updated YARN-11689: - Summary: Update the cgroup v2 init error handling to provide more straightforward error messages (was: Update getErrorWithDetails method to provide more meaningful error messages) > Update the cgroup v2 init error handling to provide more straightforward > error messages > --- > > Key: YARN-11689 > URL: https://issues.apache.org/jira/browse/YARN-11689 > Project: Hadoop YARN > Issue Type: Sub-task >Reporter: Benjamin Teke >Assignee: Benjamin Teke >Priority: Major > > The method AbstractCGroupsHandler.getErrorWithDetails hides quite a lot of > information. It would be useful to show the underlying exception and it's > message as well, by default. -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-11669) [Umbrella] cgroup v2 support
[
https://issues.apache.org/jira/browse/YARN-11669?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Benjamin Teke updated YARN-11669:
-
Description:
cgroup v2 has some fundamental changes compared to v1. RHEL9, Ubuntu 22 already
moved to cgroup v2 as a default, hence YARN should support it. This umbrella
tracks the required work.
[Documentation|https://docs.kernel.org/admin-guide/cgroup-v2.html]
A way to test the newly added features:
# Turn on cgroup v1 based on the current
[documentation|https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/NodeManagerCgroups.html].
# System prerequisites:
## the file {{/etc/mtab}} should contain a mount path with the file system type
{{cgroup2}}, by default this could be {{/sys/fs/cgroup}} on most OS's
## the {{cgroup.subtree_control}} file should contain the necessary controllers
(update it with: {{echo "+cpu +io +memory" > cgroup.subtree_control}})
## either create the YARN hierarchy and give recursive access to the user
running the NM on the node. The hierarchy is {{hadoop-yarn}} by default
(controller by
{{yarn.nodemanager.linux-container-executor.cgroups.hierarchy}}), and recursive
mode is required, because as soon as the directory is created it will be filled
with the controller files which YARN will try to edit.
### Alternatively if the NM process user has access rights on the
{{/sys/fs/cgroup}} directory it'll try to create the hierarchy and update the
{{cgroup.subtree_control}} file.
# YARN configuration
## {{yarn.nodemanager.linux-container-executor.cgroups.mount-path}} should
point to the directory where the cgroup2 structure is mounted and the
{{hadoop-yarn}} hierarchy was created
## {{yarn.nodemanager.linux-container-executor.cgroups.v2.enabled}} should be
set to {{true}}
## Enable a cgroup controller, like {{yarn. nodemanager. resource.
cpu.enabled}}: {{true}}
# Launch the NM and monitor the cgroup files on container launches (i.e:
{{/sys/fs/cgroup/hadoop-yarn/container_id/cpu.weight}})
was:
cgroup v2 has some fundamental changes compared to v1. RHEL9, Ubuntu 22 already
moved to cgroup v2 as a default, hence YARN should support it. This umbrella
tracks the required work.
[Documentation|https://docs.kernel.org/admin-guide/cgroup-v2.html]
A way to test the newly added features:
# Turn on cgroup v1 based on the current
[documentation|https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/NodeManagerCgroups.html].
# System prerequisites:
## the file {{/etc/mtab}} should contain a mount path with the file system type
{{cgroup2}}, by default this could be {{/sys/fs/cgroup}} on most OS's
## the {{cgroup.subtree_control}} file should contain the necessary controllers
(update it with: {{echo "+cpu +io +memory" > cgroup.subtree_control}})
## either create the YARN hierarchy and make it owned by the user running the
NM on the node. The hierarchy is {{hadoop-yarn}} by default (controller by
{{yarn.nodemanager.linux-container-executor.cgroups.hierarchy}}), and {{chown
-R user:group /sys/fs/cgroup/hadoop-yarn}} is needed. -R is required, because
as soon as the directory is created it will be filled with the controller files
which YARN will try to edit.
### Alternatively if the NM process user has access rights on the
{{/sys/fs/cgroup}} directory it'll try to create the hierarchy and update the
{{cgroup.subtree_control}} file.
# YARN configuration
## {{yarn.nodemanager.linux-container-executor.cgroups.mount-path}} should
point to the directory where the cgroup2 structure is mounted and the
{{hadoop-yarn}} hierarchy was created
## {{yarn.nodemanager.linux-container-executor.cgroups.v2.enabled}} should be
set to {{true}}
## Enable a cgroup controller, like {{yarn. nodemanager. resource.
cpu.enabled}}: {{true}}
# Launch the NM and monitor the cgroup files on container launches (i.e:
{{/sys/fs/cgroup/hadoop-yarn/container_id/cpu.weight}})
> [Umbrella] cgroup v2 support
>
>
> Key: YARN-11669
> URL: https://issues.apache.org/jira/browse/YARN-11669
> Project: Hadoop YARN
> Issue Type: New Feature
> Components: yarn
>Reporter: Ferenc Erdelyi
>Assignee: Benjamin Teke
>Priority: Major
>
> cgroup v2 has some fundamental changes compared to v1. RHEL9, Ubuntu 22
> already moved to cgroup v2 as a default, hence YARN should support it. This
> umbrella tracks the required work.
> [Documentation|https://docs.kernel.org/admin-guide/cgroup-v2.html]
> A way to test the newly added features:
> # Turn on cgroup v1 based on the current
> [documentation|https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/NodeManagerCgroups.html].
> # System prerequisites:
> ## the file {{/etc/mtab}} should contain a mount path with the
[jira] [Updated] (YARN-11669) [Umbrella] cgroup v2 support
[
https://issues.apache.org/jira/browse/YARN-11669?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Benjamin Teke updated YARN-11669:
-
Description:
cgroup v2 has some fundamental changes compared to v1. RHEL9, Ubuntu 22 already
moved to cgroup v2 as a default, hence YARN should support it. This umbrella
tracks the required work.
[Documentation|https://docs.kernel.org/admin-guide/cgroup-v2.html]
A way to test the newly added features:
# Turn on cgroup v1 based on the current
[documentation|https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/NodeManagerCgroups.html].
# System prerequisites:
## the file {{/etc/mtab}} should contain a mount path with the file system type
{{cgroup2}}, by default this could be {{/sys/fs/cgroup}} on most OS's
## the {{cgroup.subtree_control}} file should contain the necessary controllers
(update it with: {{echo "+cpu +io +memory" > cgroup.subtree_control}})
## either create the YARN hierarchy and make it owned by the user running the
NM on the node. The hierarchy is {{hadoop-yarn}} by default (controller by
{{yarn.nodemanager.linux-container-executor.cgroups.hierarchy}}), and {{chown
-R user:group /sys/fs/cgroup/hadoop-yarn}} is needed. -R is required, because
as soon as the directory is created it will be filled with the controller files
which YARN will try to edit.
### Alternatively if the NM process user has access rights on the
{{/sys/fs/cgroup}} directory it'll try to create the hierarchy and update the
{{cgroup.subtree_control}} file.
# YARN configuration
## {{yarn.nodemanager.linux-container-executor.cgroups.mount-path}} should
point to the directory where the cgroup2 structure is mounted and the
{{hadoop-yarn}} hierarchy was created
## {{yarn.nodemanager.linux-container-executor.cgroups.v2.enabled}} should be
set to {{true}}
## Enable a cgroup controller, like {{yarn. nodemanager. resource.
cpu.enabled}}: {{true}}
# Launch the NM and monitor the cgroup files on container launches (i.e:
{{/sys/fs/cgroup/hadoop-yarn/container_id/cpu.weight}})
was:
cgroup v2 has some fundamental changes compared to v1. RHEL9, Ubuntu 22 already
moved to cgroup v2 as a default, hence YARN should support it. This umbrella
tracks the required work.
[Documentation|https://docs.kernel.org/admin-guide/cgroup-v2.html]
A way to test the newly added features:
# Turn on cgroup v1 based on the current
[documentation|https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/NodeManagerCgroups.html].
# System prerequisites:
## the file {{/etc/mtab}} should contain a mount path with the file system type
{{cgroup2}}, by default this could be {{/sys/fs/cgroup}} on most OS's
## the {{cgroup.subtree_control}} file should contain the necessary controllers
(update it with: {{echo "+cpu +io +memory" > cgroup.subtree_control}})
## either create the YARN hierarchy and make it owned by the user running the
NM on the node. The hierarchy is {{hadoop-yarn}} by default (controller by
{{yarn.nodemanager.linux-container-executor.cgroups.hierarchy}}), and {{chown
-R yarn:hadoop /sys/fs/cgroup/hadoop-yarn}} is needed. -R is required, because
as soon as the directory is created it will be filled with the controller files
which YARN will try to edit.
### Alternatively if the NM process user has access rights on the
{{/sys/fs/cgroup}} directory it'll try to create the hierarchy and update the
{{cgroup.subtree_control}} file.
# YARN configuration
## {{yarn.nodemanager.linux-container-executor.cgroups.mount-path}} should
point to the directory where the cgroup2 structure is mounted and the
{{hadoop-yarn}} hierarchy was created
## {{yarn.nodemanager.linux-container-executor.cgroups.v2.enabled}} should be
set to {{true}}
## Enable a cgroup controller, like {{yarn. nodemanager. resource.
cpu.enabled}}: {{true}}
# Launch the NM and monitor the cgroup files on container launches (i.e:
{{/sys/fs/cgroup/hadoop-yarn/container_id/cpu.weight}})
> [Umbrella] cgroup v2 support
>
>
> Key: YARN-11669
> URL: https://issues.apache.org/jira/browse/YARN-11669
> Project: Hadoop YARN
> Issue Type: New Feature
> Components: yarn
>Reporter: Ferenc Erdelyi
>Assignee: Benjamin Teke
>Priority: Major
>
> cgroup v2 has some fundamental changes compared to v1. RHEL9, Ubuntu 22
> already moved to cgroup v2 as a default, hence YARN should support it. This
> umbrella tracks the required work.
> [Documentation|https://docs.kernel.org/admin-guide/cgroup-v2.html]
> A way to test the newly added features:
> # Turn on cgroup v1 based on the current
> [documentation|https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/NodeManagerCgroups.html].
> # System prerequisites:
> ## the file {{/etc/mtab}} sh
[jira] [Updated] (YARN-11669) [Umbrella] cgroup v2 support
[
https://issues.apache.org/jira/browse/YARN-11669?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Benjamin Teke updated YARN-11669:
-
Description:
cgroup v2 has some fundamental changes compared to v1. RHEL9, Ubuntu 22 already
moved to cgroup v2 as a default, hence YARN should support it. This umbrella
tracks the required work.
[Documentation|https://docs.kernel.org/admin-guide/cgroup-v2.html]
A way to test the newly added features:
# Turn on cgroup v1 based on the current
[documentation|https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/NodeManagerCgroups.html].
# System prerequisites:
## the file {{/etc/mtab}} should contain a mount path with the file system type
{{cgroup2}}, by default this could be {{/sys/fs/cgroup}} on most OS's
## the {{cgroup.subtree_control}} file should contain the necessary controllers
(update it with: {{echo "+cpu +io +memory" > cgroup.subtree_control}})
## either create the YARN hierarchy and make it owned by the user running the
NM on the node. The hierarchy is {{hadoop-yarn}} by default (controller by
{{yarn.nodemanager.linux-container-executor.cgroups.hierarchy}}), and {{chown
-R yarn:hadoop /sys/fs/cgroup/hadoop-yarn}} is needed. -R is required, because
as soon as the directory is created it will be filled with the controller files
which YARN will try to edit.
### Alternatively if the NM process user has access rights on the
{{/sys/fs/cgroup}} directory it'll try to create the hierarchy and update the
{{cgroup.subtree_control}} file.
# YARN configuration
## {{yarn.nodemanager.linux-container-executor.cgroups.mount-path}} should
point to the directory where the cgroup2 structure is mounted and the
{{hadoop-yarn}} hierarchy was created
## {{yarn.nodemanager.linux-container-executor.cgroups.v2.enabled}} should be
set to {{true}}
## Enable a cgroup controller, like {{yarn. nodemanager. resource.
cpu.enabled}}: {{true}}
# Launch the NM and monitor the cgroup files on container launches (i.e:
{{/sys/fs/cgroup/hadoop-yarn/container_id/cpu.weight}})
was:
cgroup v2 has some fundamental changes compared to v1. RHEL9, Ubuntu 22 already
moved to cgroup v2 as a default, hence YARN should support it. This umbrella
tracks the required work.
[Documentation|https://docs.kernel.org/admin-guide/cgroup-v2.html]
> [Umbrella] cgroup v2 support
>
>
> Key: YARN-11669
> URL: https://issues.apache.org/jira/browse/YARN-11669
> Project: Hadoop YARN
> Issue Type: New Feature
> Components: yarn
>Reporter: Ferenc Erdelyi
>Assignee: Benjamin Teke
>Priority: Major
>
> cgroup v2 has some fundamental changes compared to v1. RHEL9, Ubuntu 22
> already moved to cgroup v2 as a default, hence YARN should support it. This
> umbrella tracks the required work.
> [Documentation|https://docs.kernel.org/admin-guide/cgroup-v2.html]
> A way to test the newly added features:
> # Turn on cgroup v1 based on the current
> [documentation|https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/NodeManagerCgroups.html].
> # System prerequisites:
> ## the file {{/etc/mtab}} should contain a mount path with the file system
> type {{cgroup2}}, by default this could be {{/sys/fs/cgroup}} on most OS's
> ## the {{cgroup.subtree_control}} file should contain the necessary
> controllers (update it with: {{echo "+cpu +io +memory" >
> cgroup.subtree_control}})
> ## either create the YARN hierarchy and make it owned by the user running the
> NM on the node. The hierarchy is {{hadoop-yarn}} by default (controller by
> {{yarn.nodemanager.linux-container-executor.cgroups.hierarchy}}), and {{chown
> -R yarn:hadoop /sys/fs/cgroup/hadoop-yarn}} is needed. -R is required,
> because as soon as the directory is created it will be filled with the
> controller files which YARN will try to edit.
> ### Alternatively if the NM process user has access rights on the
> {{/sys/fs/cgroup}} directory it'll try to create the hierarchy and update the
> {{cgroup.subtree_control}} file.
> # YARN configuration
> ## {{yarn.nodemanager.linux-container-executor.cgroups.mount-path}} should
> point to the directory where the cgroup2 structure is mounted and the
> {{hadoop-yarn}} hierarchy was created
> ## {{yarn.nodemanager.linux-container-executor.cgroups.v2.enabled}} should be
> set to {{true}}
> ## Enable a cgroup controller, like {{yarn. nodemanager. resource.
> cpu.enabled}}: {{true}}
> # Launch the NM and monitor the cgroup files on container launches (i.e:
> {{/sys/fs/cgroup/hadoop-yarn/container_id/cpu.weight}})
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11687) Update CGroupsResourceCalculator to track usages using cgroupv2
[
https://issues.apache.org/jira/browse/YARN-11687?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17845384#comment-17845384
]
ASF GitHub Bot commented on YARN-11687:
---
brumi1024 commented on code in PR #6780:
URL: https://github.com/apache/hadoop/pull/6780#discussion_r1596811423
##
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java:
##
@@ -18,258 +18,138 @@
package
org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.util.ControlledClock;
-import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
-import org.junit.Assert;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.stream.Collectors;
+
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
-import java.io.File;
-import java.nio.charset.StandardCharsets;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.util.CpuTimeTracker;
+import org.apache.hadoop.yarn.exceptions.YarnException;
-import static org.mockito.Mockito.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
/**
* Unit test for CGroupsResourceCalculator.
*/
public class TestCGroupsResourceCalculator {
- private ControlledClock clock = new ControlledClock();
- private CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class);
- private String basePath = "/tmp/" + this.getClass().getName();
+ private Path root;
- public TestCGroupsResourceCalculator() {
-when(cGroupsHandler.getRelativePathForCGroup("container_1"))
-.thenReturn("/yarn/container_1");
-when(cGroupsHandler.getRelativePathForCGroup("")).thenReturn("/yarn/");
+ @Before
+ public void before() throws IOException {
+root = Files.createTempDirectory("TestCGroupsResourceCalculator");
+ }
+
+ @After
+ public void after() throws IOException {
+FileUtils.deleteDirectory(root.toFile());
}
- @Test(expected = YarnException.class)
+ @Test(expected = IOException.class)
public void testPidNotFound() throws Exception {
-CGroupsResourceCalculator calculator =
-new CGroupsResourceCalculator(
-"1234", ".", cGroupsHandler, clock, 10);
+CGroupsResourceCalculator calculator = createCalculator();
calculator.setCGroupFilePaths();
-Assert.assertEquals("Expected exception", null, calculator);
}
- @Test(expected = YarnException.class)
+ @Test
public void testNoMemoryCGgroupMount() throws Exception {
-File procfs = new File(basePath + "/1234");
-Assert.assertTrue("Setup error", procfs.mkdirs());
+writeToFile("proc/41/cgroup",
+"7:devices:/yarn/container_1",
+"6:cpuacct,cpu:/yarn/container_1",
+"5:pids:/yarn/container_1"
+);
+
+CGroupsResourceCalculator calculator = createCalculator();
try {
- FileUtils.writeStringToFile(
- new File(procfs, CGroupsResourceCalculator.CGROUP),
- "7:devices:/yarn/container_1\n" +
- "6:cpuacct,cpu:/yarn/container_1\n" +
- "5:pids:/yarn/container_1\n", StandardCharsets.UTF_8);
- CGroupsResourceCalculator calculator =
- new CGroupsResourceCalculator(
- "1234", basePath,
- cGroupsHandler, clock, 10);
calculator.setCGroupFilePaths();
- Assert.assertEquals("Expected exception", null, calculator);
-} finally {
- FileUtils.deleteDirectory(new File(basePath));
+ fail("No exception was thrown");
+} catch (YarnException e) {
+ assertEquals("Controller MEMORY not found for 41 pid", e.getMessage());
Review Comment:
Nit: it might be a bit more developer friendly to assert a contains(MEMORY)
and contains(pid), I wouldn't expect test failures for message wording changes.
> Update CGroupsResourceCalculator to track usages using cgroupv2
> ---
>
> Key: YARN-11687
> URL: https://issues.apache.org/jira/browse/YARN-11687
> Project: Hadoop YARN
> Issue Type: Sub-task
>Reporter: Benjamin Teke
>Assignee: Bence Kosztolnik
>Priority: Major
> Labels
[jira] [Commented] (YARN-11687) Update CGroupsResourceCalculator to track usages using cgroupv2
[
https://issues.apache.org/jira/browse/YARN-11687?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17845385#comment-17845385
]
ASF GitHub Bot commented on YARN-11687:
---
brumi1024 commented on code in PR #6780:
URL: https://github.com/apache/hadoop/pull/6780#discussion_r1596818926
##
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMixedResourceCalculator.java:
##
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package
org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
+
+/**
+ * In case if {@link CGroupsV2ResourceCalculator} can provide the required
info that will be used.
+ * Otherwise, it will fall back to the {@link CGroupsResourceCalculator}
implementation.
+ * Similar like {@link CombinedResourceCalculator} works.
+ */
+public class CGroupsMixedResourceCalculator extends
ResourceCalculatorProcessTree {
Review Comment:
Sorry, just to complicate things a bit more: there seems to be a
CombinedResourceCalculator which takes cgroup and procfs based
resourcecalculators into account. But the change seems small, probably it's
CGroupsResourceCalculator cgroup should use this class instead.
> Update CGroupsResourceCalculator to track usages using cgroupv2
> ---
>
> Key: YARN-11687
> URL: https://issues.apache.org/jira/browse/YARN-11687
> Project: Hadoop YARN
> Issue Type: Sub-task
>Reporter: Benjamin Teke
>Assignee: Bence Kosztolnik
>Priority: Major
> Labels: pull-request-available
>
> [CGroupsResourceCalculator|https://github.com/apache/hadoop/blob/f609460bda0c2bd87dd3580158e549e2f34f14d5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java]
> should also be updated to handle the cgroup v2 changes.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11687) Update CGroupsResourceCalculator to track usages using cgroupv2
[
https://issues.apache.org/jira/browse/YARN-11687?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17845374#comment-17845374
]
ASF GitHub Bot commented on YARN-11687:
---
brumi1024 commented on code in PR #6780:
URL: https://github.com/apache/hadoop/pull/6780#discussion_r1596818926
##
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMixedResourceCalculator.java:
##
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package
org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
+
+/**
+ * In case if {@link CGroupsV2ResourceCalculator} can provide the required
info that will be used.
+ * Otherwise, it will fall back to the {@link CGroupsResourceCalculator}
implementation.
+ * Similar like {@link CombinedResourceCalculator} works.
+ */
+public class CGroupsMixedResourceCalculator extends
ResourceCalculatorProcessTree {
Review Comment:
Sorry, just to complicate things a bit more: there seems to be a
CombinedResourceCalculator which takes cgroup and procfs based
resourcecalculators into account. Probably it's CGroupsResourceCalculator
cgroup should use this class instead.
##
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java:
##
@@ -18,258 +18,138 @@
package
org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.util.ControlledClock;
-import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
-import org.junit.Assert;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.stream.Collectors;
+
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
-import java.io.File;
-import java.nio.charset.StandardCharsets;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.util.CpuTimeTracker;
+import org.apache.hadoop.yarn.exceptions.YarnException;
-import static org.mockito.Mockito.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
/**
* Unit test for CGroupsResourceCalculator.
*/
public class TestCGroupsResourceCalculator {
- private ControlledClock clock = new ControlledClock();
- private CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class);
- private String basePath = "/tmp/" + this.getClass().getName();
+ private Path root;
- public TestCGroupsResourceCalculator() {
-when(cGroupsHandler.getRelativePathForCGroup("container_1"))
-.thenReturn("/yarn/container_1");
-when(cGroupsHandler.getRelativePathForCGroup("")).thenReturn("/yarn/");
+ @Before
+ public void before() throws IOException {
+root = Files.createTempDirectory("TestCGroupsResourceCalculator");
+ }
+
+ @After
+ public void after() throws IOException {
+FileUtils.deleteDirectory(root.toFile());
}
- @Test(expected = YarnException.class)
+ @Test(expected = IOException.class)
public void testPidNotFound() throws Exception {
-CGroupsResourceCalculator calculator =
-new CGroupsResourceCalculator(
-"1234", ".", cGroupsHandler, clock, 10);
+CGroupsResourceCalculator calculator = createCalculator();
calculator.setCGroupFilePaths();
-Assert.assertEquals("Expected exception", null, calculator);
}
- @Test(expected = YarnExce
