This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new a96f399d5094 [SPARK-46883][CORE][FOLLOWUP] Fix `clusterutilization`
API to handle 0 worker case
a96f399d5094 is described below
commit a96f399d5094a3473ffc0e55390105d013a3d22f
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Sat Jan 27 19:02:37 2024 -0800
[SPARK-46883][CORE][FOLLOWUP] Fix `clusterutilization` API to handle 0
worker case
### What changes were proposed in this pull request?
This PR is a follow-up of #44908 to fix `clusterutilization` API to handle
0 worker case.
### Why are the changes needed?
To fix `ArithmeticException`
```
$ curl http://localhost:8080/json/clusterutilization
<html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 java.lang.ArithmeticException: / by zero</title>
</head>
<body><h2>HTTP ERROR 500 java.lang.ArithmeticException: / by zero</h2>
<table>
<tr><th>URI:</th><td>/json/clusterutilization</td></tr>
```
### Does this PR introduce _any_ user-facing change?
No, this feature and bug is not released yet.
### How was this patch tested?
Pass the CIs with the newly added test case.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #44914 from dongjoon-hyun/SPARK-46883-2.
Lead-authored-by: Dongjoon Hyun <[email protected]>
Co-authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../org/apache/spark/deploy/JsonProtocol.scala | 4 ++--
.../apache/spark/deploy/JsonProtocolSuite.scala | 24 +++++++++++++++++++++-
2 files changed, 25 insertions(+), 3 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala
b/core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala
index 9c73e84f4166..04302c77a398 100644
--- a/core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala
@@ -312,9 +312,9 @@ private[deploy] object JsonProtocol {
("waitingDrivers" -> obj.activeDrivers.count(_.state ==
DriverState.SUBMITTED)) ~
("cores" -> cores) ~
("coresused" -> coresUsed) ~
- ("coresutilization" -> 100 * coresUsed / cores) ~
+ ("coresutilization" -> (if (cores == 0) 100 else 100 * coresUsed / cores))
~
("memory" -> memory) ~
("memoryused" -> memoryUsed) ~
- ("memoryutilization" -> 100 * memoryUsed / memory)
+ ("memoryutilization" -> (if (memory == 0) 100 else 100 * memoryUsed /
memory))
}
}
diff --git
a/core/src/test/scala/org/apache/spark/deploy/JsonProtocolSuite.scala
b/core/src/test/scala/org/apache/spark/deploy/JsonProtocolSuite.scala
index 6fca31234ee2..518a8c8b3d05 100644
--- a/core/src/test/scala/org/apache/spark/deploy/JsonProtocolSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/JsonProtocolSuite.scala
@@ -25,7 +25,7 @@ import org.json4s.jackson.JsonMethods
import org.apache.spark.{JsonTestUtils, SparkFunSuite}
import org.apache.spark.deploy.DeployMessages.{MasterStateResponse,
WorkerStateResponse}
-import org.apache.spark.deploy.master.{ApplicationInfo, RecoveryState}
+import org.apache.spark.deploy.master.{ApplicationInfo, RecoveryState,
WorkerInfo}
import org.apache.spark.deploy.worker.ExecutorRunner
class JsonProtocolSuite extends SparkFunSuite with JsonTestUtils {
@@ -119,6 +119,21 @@ class JsonProtocolSuite extends SparkFunSuite with
JsonTestUtils {
assertValidDataInJson(output,
JsonMethods.parse(JsonConstants.clusterUtilizationJsonStr))
}
+ test("SPARK-46883: writeClusterUtilization without workers") {
+ val workers = Array.empty[WorkerInfo]
+ val activeApps = Array(createAppInfo())
+ val completedApps = Array.empty[ApplicationInfo]
+ val activeDrivers = Array(createDriverInfo())
+ val completedDrivers = Array(createDriverInfo())
+ val stateResponse = new MasterStateResponse(
+ "host", 8080, None, workers, activeApps, completedApps,
+ activeDrivers, completedDrivers, RecoveryState.ALIVE)
+ val output = JsonProtocol.writeClusterUtilization(stateResponse)
+ assertValidJson(output)
+ assertValidDataInJson(output,
+ JsonMethods.parse(JsonConstants.clusterUtilizationWithoutWorkersJsonStr))
+ }
+
def assertValidJson(json: JValue): Unit = {
try {
JsonMethods.parse(JsonMethods.compact(json))
@@ -227,4 +242,11 @@ object JsonConstants {
|"cores":8,"coresused":0,"coresutilization":0,
|"memory":2468,"memoryused":0,"memoryutilization":0}
""".stripMargin
+
+ val clusterUtilizationWithoutWorkersJsonStr =
+ """
+ |{"waitingDrivers":1,
+ |"cores":0,"coresused":0,"coresutilization":100,
+ |"memory":0,"memoryused":0,"memoryutilization":100}
+ """.stripMargin
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
