This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new 6e443658991 [SPARK-43471][CORE] Handle missing hadoopProperties and
metricsProperties
6e443658991 is described below
commit 6e443658991b2596466a92433cca4bb6010861e4
Author: Dongjoon Hyun <dongj...@apache.org>
AuthorDate: Thu May 11 15:30:04 2023 -0700
[SPARK-43471][CORE] Handle missing hadoopProperties and metricsProperties
### What changes were proposed in this pull request?
This PR aims to handle a corner case where `hadoopProperties` and
`metricsProperties` is null which means not loaded.
### Why are the changes needed?
To prevent NPE.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass the CIs with the newly added test suite.
Closes #41145 from TQJADE/SPARK-43471.
Lead-authored-by: Dongjoon Hyun <dongj...@apache.org>
Co-authored-by: Qi Tan <qi_...@apple.com>
Signed-off-by: Dongjoon Hyun <dongj...@apache.org>
(cherry picked from commit 1dba7b803ecffd09c544009b79a6a3219f56d4e0)
Signed-off-by: Dongjoon Hyun <dongj...@apache.org>
---
.../org/apache/spark/ui/env/EnvironmentPage.scala | 9 ++--
.../apache/spark/ui/env/EnvironmentPageSuite.scala | 53 ++++++++++++++++++++++
2 files changed, 58 insertions(+), 4 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala
b/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala
index c6e224732cb..4aaa04019cc 100644
--- a/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala
@@ -77,15 +77,16 @@ private[ui] class EnvironmentPage(
val sparkPropertiesTable = UIUtils.listingTable(propertyHeader,
propertyRow,
Utils.redact(conf, appEnv.sparkProperties.sorted), fixedWidth = true,
headerClasses = headerClasses)
+ val emptyProperties = collection.Seq.empty[(String, String)]
val hadoopPropertiesTable = UIUtils.listingTable(propertyHeader,
propertyRow,
- Utils.redact(conf, appEnv.hadoopProperties.sorted), fixedWidth = true,
- headerClasses = headerClasses)
+ Utils.redact(conf,
Option(appEnv.hadoopProperties).getOrElse(emptyProperties).sorted),
+ fixedWidth = true, headerClasses = headerClasses)
val systemPropertiesTable = UIUtils.listingTable(propertyHeader,
propertyRow,
Utils.redact(conf, appEnv.systemProperties.sorted), fixedWidth = true,
headerClasses = headerClasses)
val metricsPropertiesTable = UIUtils.listingTable(propertyHeader,
propertyRow,
- Utils.redact(conf, appEnv.metricsProperties.sorted), fixedWidth = true,
- headerClasses = headerClasses)
+ Utils.redact(conf,
Option(appEnv.metricsProperties).getOrElse(emptyProperties).sorted),
+ fixedWidth = true, headerClasses = headerClasses)
val classpathEntriesTable = UIUtils.listingTable(
classPathHeader, classPathRow, appEnv.classpathEntries.sorted,
fixedWidth = true,
headerClasses = headerClasses)
diff --git
a/core/src/test/scala/org/apache/spark/ui/env/EnvironmentPageSuite.scala
b/core/src/test/scala/org/apache/spark/ui/env/EnvironmentPageSuite.scala
new file mode 100644
index 00000000000..92791874668
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/ui/env/EnvironmentPageSuite.scala
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ui.env
+
+import javax.servlet.http.HttpServletRequest
+
+import org.mockito.Mockito._
+
+import org.apache.spark.SparkConf
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.status.AppStatusStore
+import org.apache.spark.status.api.v1.{ApplicationEnvironmentInfo, RuntimeInfo}
+
+class EnvironmentPageSuite extends SparkFunSuite {
+
+ test("SPARK-43471: Handle missing hadoopProperties and metricsProperties") {
+ val environmentTab = mock(classOf[EnvironmentTab])
+ when(environmentTab.appName).thenReturn("Environment")
+ when(environmentTab.basePath).thenReturn("http://localhost:4040";)
+ when(environmentTab.headerTabs).thenReturn(Seq.empty)
+
+ val runtimeInfo = mock(classOf[RuntimeInfo])
+
+ val info = mock(classOf[ApplicationEnvironmentInfo])
+ when(info.runtime).thenReturn(runtimeInfo)
+ when(info.sparkProperties).thenReturn(Seq.empty)
+ when(info.systemProperties).thenReturn(Seq.empty)
+ when(info.classpathEntries).thenReturn(Seq.empty)
+
+ val store = mock(classOf[AppStatusStore])
+ when(store.environmentInfo).thenReturn(info)
+ when(store.resourceProfileInfo).thenReturn(Seq.empty)
+
+ val environmentPage = new EnvironmentPage(environmentTab, new SparkConf,
store)
+ val request = mock(classOf[HttpServletRequest])
+ environmentPage.render(request)
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
