This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 9201cc79b [MINOR] Update log level of PushData and PushMergedData for
PushDataHandler from info to debug
9201cc79b is described below
commit 9201cc79b0519a0eaf348e35afb047a8acd7aab7
Author: SteNicholas <[email protected]>
AuthorDate: Tue Jan 9 15:27:50 2024 +0800
[MINOR] Update log level of PushData and PushMergedData for PushDataHandler
from info to debug
### What changes were proposed in this pull request?
Update log level of `PushData` and `PushMergedData` for `PushDataHandler`
from info to debug.
### Why are the changes needed?
In production practice, there are many logs for
`PushDataHandler#handlePushData` and `PushDataHandler#handlePushMergedData`,
which are unnecessary to use info level to print.
```
# cat worker-stderr.log.2024_01_09_08 |grep "INFO"|grep "hard split"|wc -l
10914811
```
```
2024-01-09 11:01:16,082 [INFO] [push-server-6-37] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1650016801129_24552188_1-1, map 40707 attempt 0)
2024-01-09 11:01:16,082 [INFO] [push-server-6-10] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1703497903483_735656_1-1, map 23739 attempt 0)
2024-01-09 11:01:16,083 [INFO] [push-server-6-37] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1650016801129_24552188_1-1, map 40775 attempt 0)
2024-01-09 11:01:16,084 [INFO] [push-server-6-25] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1650016801129_24552188_1-1, map 41286 attempt 0)
2024-01-09 11:01:16,085 [INFO] [push-server-6-19] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1703497903483_735656_1-1, map 24123 attempt 0)
2024-01-09 11:01:16,085 [INFO] [push-server-6-37] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1650016801129_24552188_1-1, map 41429 attempt 0)
2024-01-09 11:01:16,086 [INFO] [push-server-6-8] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1650016801129_24552188_1-1, map 41512 attempt 0)
2024-01-09 11:01:16,088 [INFO] [push-server-6-59] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1650016801129_24552188_1-1, map 41512 attempt 0)
2024-01-09 11:01:16,088 [INFO] [push-server-6-21] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1703497903483_735656_1-1, map 24075 attempt 0)
2024-01-09 11:01:16,088 [INFO] [push-server-6-8] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1650016801129_24552188_1-1, map 41512 attempt 0)
2024-01-09 11:01:16,090 [INFO] [push-server-6-21] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1703497903483_735656_1-1, map 24075 attempt 0)
2024-01-09 11:01:16,090 [INFO] [push-server-6-35] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1650016801129_24552188_1-1, map 41523 attempt 0)
2024-01-09 11:01:16,090 [INFO] [push-server-6-62] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1703497903483_735656_1-1, map 22060 attempt 0)
2024-01-09 11:01:16,091 [INFO] [push-server-6-56] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1703497903483_735656_1-1, map 21910 attempt 0)
2024-01-09 11:01:16,091 [INFO] [push-server-6-55] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1703497903483_735656_1-1, map 21910 attempt 0)
2024-01-09 11:01:16,092 [INFO] [push-server-6-56] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1703497903483_735656_1-1, map 21910 attempt 0)
2024-01-09 11:01:16,092 [INFO] [push-server-6-55] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1703497903483_735656_1-1, map 21910 attempt 0)
2024-01-09 11:01:16,092 [INFO] [push-server-6-55] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1703497903483_735656_1-1, map 21910 attempt 0)
2024-01-09 11:01:16,092 [INFO] [push-server-6-56] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1703497903483_735656_1-1, map 21910 attempt 0)
2024-01-09 11:01:16,092 [INFO] [push-server-6-23] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1703497903483_735656_1-1, map 21747 attempt 0)
2024-01-09 11:01:16,092 [INFO] [push-server-6-56] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1703497903483_735656_1-1, map 21910 attempt 0)
2024-01-09 11:01:16,093 [INFO] [push-server-6-56] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1703497903483_735656_1-1, map 21910 attempt 0)
2024-01-09 11:01:16,093 [INFO] [push-server-6-21] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1703497903483_735656_1-1, map 21329 attempt 0)
2024-01-09 11:01:16,093 [INFO] [push-server-6-23] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1703497903483_735656_1-1, map 21747 attempt 0)
2024-01-09 11:01:16,093 [INFO] [push-server-6-21] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1703497903483_735656_1-1, map 21327 attempt 0)
2024-01-09 11:01:16,093 [INFO] [push-server-6-23] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1703497903483_735656_1-1, map 21747 attempt 0)
2024-01-09 11:01:16,094 [INFO] [push-server-6-41] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1703497903483_735656_1-1, map 23648 attempt 0)
2024-01-09 11:01:16,095 [INFO] [push-server-6-41] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1703497903483_735656_1-1, map 21747 attempt 0)
2024-01-09 11:01:16,096 [INFO] [push-server-6-22] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1650016801129_24552188_1-1, map 41716 attempt 0)
2024-01-09 11:01:16,096 [INFO] [push-server-6-8] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1650016801129_24552188_1-1, map 40178 attempt 0)
2024-01-09 11:01:16,096 [INFO] [push-server-6-23] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1703497903483_735656_1-1, map 23648 attempt 0)
2024-01-09 11:01:16,097 [INFO] [push-server-6-2] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1703497903483_735656_1-1, map 21329 attempt 0)
2024-01-09 11:01:16,098 [INFO] [push-server-6-56] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1703497903483_735656_1-1, map 21910 attempt 0)
2024-01-09 11:01:16,098 [INFO] [push-server-6-23] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1703497903483_735656_1-1, map 23827 attempt 1)
2024-01-09 11:01:16,099 [INFO] [push-server-6-41] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1703497903483_735656_1-1, map 23827 attempt 1)
2024-01-09 11:01:16,099 [INFO] [push-server-6-14] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1650016801129_24552188_1-1, map 40057 attempt 0)
2024-01-09 11:01:16,103 [INFO] [push-server-6-8] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1650016801129_24552188_1-1, map 41276 attempt 0)
2024-01-09 11:01:16,104 [INFO] [push-server-6-55] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1703497903483_735656_1-1, map 22326 attempt 0)
2024-01-09 11:01:16,105 [INFO] [push-server-6-37] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1650016801129_24552188_1-1, map 40371 attempt 0)
2024-01-09 11:01:16,106 [INFO] [push-server-6-14] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1703497903483_735656_1-1, map 22140 attempt 0)
2024-01-09 11:01:16,106 [INFO] [push-server-6-19] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1650016801129_24552188_1-1, map 41825 attempt 0)
2024-01-09 11:01:16,109 [INFO] [push-server-6-5] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1703497903483_735656_1-1, map 22033 attempt 0)
2024-01-09 11:01:16,109 [INFO] [push-server-6-53] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1703497903483_735656_1-1, map 22033 attempt 0)
2024-01-09 11:01:16,109 [INFO] [push-server-6-21] -
org.apache.celeborn.service.deploy.worker.PushDataHandler -Logging.scala(51)
-[Case1] Receive push merged data for committed hard split partition of
(shuffle application_1703497903483_735656_1-1, map 21327 attempt 0)
```
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
No.
Closes #2215 from SteNicholas/handle-push-data-log.
Authored-by: SteNicholas <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
---
.../celeborn/service/deploy/worker/PushDataHandler.scala | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/PushDataHandler.scala
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/PushDataHandler.scala
index bbd5a2087..553d4d25d 100644
---
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/PushDataHandler.scala
+++
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/PushDataHandler.scala
@@ -202,12 +202,12 @@ class PushDataHandler(val workerSource: WorkerSource)
extends BaseMessageHandler
if (shuffleMapperAttempts.containsKey(shuffleKey)) {
if (-1 != shuffleMapperAttempts.get(shuffleKey).get(mapId)) {
// partition data has already been committed
- logInfo(
+ logDebug(
s"[Case1] Receive push data from speculative task(shuffle
$shuffleKey, map $mapId, " +
s" attempt $attemptId), but this mapper has already been ended.")
callbackWithTimer.onSuccess(ByteBuffer.wrap(Array[Byte](StatusCode.MAP_ENDED.getValue)))
} else {
- logInfo(
+ logDebug(
s"Receive push data for committed hard split partition of (shuffle
$shuffleKey, " +
s"map $mapId attempt $attemptId)")
callbackWithTimer.onSuccess(ByteBuffer.wrap(Array[Byte](StatusCode.HARD_SPLIT.getValue)))
@@ -217,7 +217,7 @@ class PushDataHandler(val workerSource: WorkerSource)
extends BaseMessageHandler
// If there is no shuffle key in shuffleMapperAttempts but there is
shuffle key
// in StorageManager. This partition should be HARD_SPLIT partition
and
// after worker restart, some tasks still push data to this
HARD_SPLIT partition.
- logInfo(s"[Case2] Receive push data for committed hard split
partition of " +
+ logDebug(s"[Case2] Receive push data for committed hard split
partition of " +
s"(shuffle $shuffleKey, map $mapId attempt $attemptId)")
callbackWithTimer.onSuccess(ByteBuffer.wrap(Array[Byte](StatusCode.HARD_SPLIT.getValue)))
} else {
@@ -466,13 +466,13 @@ class PushDataHandler(val workerSource: WorkerSource)
extends BaseMessageHandler
// it's probably because commitFiles for HARD_SPLIT happens.
if (shuffleMapperAttempts.containsKey(shuffleKey)) {
if (-1 != shuffleMapperAttempts.get(shuffleKey).get(mapId)) {
- logInfo(s"Receive push merged data from speculative " +
+ logDebug(s"Receive push merged data from speculative " +
s"task(shuffle $shuffleKey, map $mapId, attempt $attemptId), " +
s"but this mapper has already been ended.")
callbackWithTimer.onSuccess(
ByteBuffer.wrap(Array[Byte](StatusCode.MAP_ENDED.getValue)))
} else {
- logInfo(s"[Case1] Receive push merged data for committed hard
split partition of " +
+ logDebug(s"[Case1] Receive push merged data for committed hard
split partition of " +
s"(shuffle $shuffleKey, map $mapId attempt $attemptId)")
callbackWithTimer.onSuccess(
ByteBuffer.wrap(Array[Byte](StatusCode.HARD_SPLIT.getValue)))
@@ -482,7 +482,7 @@ class PushDataHandler(val workerSource: WorkerSource)
extends BaseMessageHandler
// If there is no shuffle key in shuffleMapperAttempts but there
is shuffle key
// in StorageManager. This partition should be HARD_SPLIT
partition and
// after worker restart, some tasks still push data to this
HARD_SPLIT partition.
- logInfo(s"[Case2] Receive push merged data for committed hard
split partition of " +
+ logDebug(s"[Case2] Receive push merged data for committed hard
split partition of " +
s"(shuffle $shuffleKey, map $mapId attempt $attemptId)")
callbackWithTimer.onSuccess(
ByteBuffer.wrap(Array[Byte](StatusCode.HARD_SPLIT.getValue)))
