This is an automated email from the ASF dual-hosted git repository.
gengliang pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 0058212 [SPARK-31584][WEBUI] Fix NullPointerException when parsing
event log with InMemoryStore
0058212 is described below
commit 0058212dc60129357ad90e2c04bcb22094a9dd84
Author: Baohe Zhang <baohe.zh...@verizonmedia.com>
AuthorDate: Tue Apr 28 17:27:13 2020 -0700
[SPARK-31584][WEBUI] Fix NullPointerException when parsing event log with
InMemoryStore
### What changes were proposed in this pull request?
https://github.com/apache/spark/pull/27716 introduced parent index for
InMemoryStore. When the method "deleteParentIndex(Object key)" in
InMemoryStore.java is called and the key is not contained in "NaturalKeys v",
A java.lang.NullPointerException will be thrown. This patch fixed the issue by
updating the if condition.
### Why are the changes needed?
Fixed a minor bug.
### Does this PR introduce any user-facing change?
No.
### How was this patch tested?
Added a unit test for deleteParentIndex.
Closes #28378 from baohe-zhang/SPARK-31584.
Authored-by: Baohe Zhang <baohe.zh...@verizonmedia.com>
Signed-off-by: Gengliang Wang <gengliang.w...@databricks.com>
(cherry picked from commit 3808014a2fa763866ad55776ccb397de4d13afcc)
Signed-off-by: Gengliang Wang <gengliang.w...@databricks.com>
---
.../apache/spark/util/kvstore/InMemoryStore.java | 2 +-
.../org/apache/spark/util/kvstore/CustomType2.java | 50 ++++++++++++++++++++++
.../spark/util/kvstore/InMemoryStoreSuite.java | 42 ++++++++++++++++++
3 files changed, 93 insertions(+), 1 deletion(-)
diff --git
a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java
b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java
index e929c6c..42e090b 100644
---
a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java
+++
b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java
@@ -293,7 +293,7 @@ public class InMemoryStore implements KVStore {
private void deleteParentIndex(Object key) {
if (hasNaturalParentIndex) {
for (NaturalKeys v : parentToChildrenMap.values()) {
- if (v.remove(asKey(key))) {
+ if (v.remove(asKey(key)) != null) {
// `v` can be empty after removing the natural key and we can
remove it from
// `parentToChildrenMap`. However, `parentToChildrenMap` is a
ConcurrentMap and such
// checking and deleting can be slow.
diff --git
a/common/kvstore/src/test/java/org/apache/spark/util/kvstore/CustomType2.java
b/common/kvstore/src/test/java/org/apache/spark/util/kvstore/CustomType2.java
new file mode 100644
index 0000000..3bb66bb
--- /dev/null
+++
b/common/kvstore/src/test/java/org/apache/spark/util/kvstore/CustomType2.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.util.kvstore;
+
+public class CustomType2 {
+
+ @KVIndex(parent = "parentId")
+ public String key;
+
+ @KVIndex("id")
+ public String id;
+
+ @KVIndex("parentId")
+ public String parentId;
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof CustomType2) {
+ CustomType2 other = (CustomType2) o;
+ return id.equals(other.id) && parentId.equals(other.parentId);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return id.hashCode() ^ parentId.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return "CustomType2[key=" + key + ",id=" + id + ",parentId=" + parentId;
+ }
+
+}
diff --git
a/common/kvstore/src/test/java/org/apache/spark/util/kvstore/InMemoryStoreSuite.java
b/common/kvstore/src/test/java/org/apache/spark/util/kvstore/InMemoryStoreSuite.java
index da52676..35656fb 100644
---
a/common/kvstore/src/test/java/org/apache/spark/util/kvstore/InMemoryStoreSuite.java
+++
b/common/kvstore/src/test/java/org/apache/spark/util/kvstore/InMemoryStoreSuite.java
@@ -210,4 +210,46 @@ public class InMemoryStoreSuite {
assertFalse(store.view(t1.getClass()).first(t2.id).skip(1).iterator().hasNext());
}
+ @Test
+ public void testDeleteParentIndex() throws Exception {
+ KVStore store = new InMemoryStore();
+
+ CustomType2 t1 = new CustomType2();
+ t1.key = "key1";
+ t1.id = "id1";
+ t1.parentId = "parentId1";
+ store.write(t1);
+
+ CustomType2 t2 = new CustomType2();
+ t2.key = "key2";
+ t2.id = "id2";
+ t2.parentId = "parentId1";
+ store.write(t2);
+
+ CustomType2 t3 = new CustomType2();
+ t3.key = "key3";
+ t3.id = "id1";
+ t3.parentId = "parentId2";
+ store.write(t3);
+
+ CustomType2 t4 = new CustomType2();
+ t4.key = "key4";
+ t4.id = "id2";
+ t4.parentId = "parentId2";
+ store.write(t4);
+
+ assertEquals(4, store.count(CustomType2.class));
+
+ store.delete(t1.getClass(), t1.key);
+ assertEquals(3, store.count(CustomType2.class));
+
+ store.delete(t2.getClass(), t2.key);
+ assertEquals(2, store.count(CustomType2.class));
+
+ store.delete(t3.getClass(), t3.key);
+ assertEquals(1, store.count(CustomType2.class));
+
+ store.delete(t4.getClass(), t4.key);
+ assertEquals(0, store.count(CustomType2.class));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
