This is an automated email from the ASF dual-hosted git repository.
zjffdu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/master by this push:
new fa587cf [ZEPPELIN-5254]. Memory leak of zeppelin server
fa587cf is described below
commit fa587cf35f64c1c81097523ef2ff0b0f6f7254b2
Author: Jeff Zhang <[email protected]>
AuthorDate: Wed Feb 17 22:57:07 2021 +0800
[ZEPPELIN-5254]. Memory leak of zeppelin server
### What is this PR for?
This PR address the memory leak in zeppelin server process, there're 3
kinds of objects that have leak issue
* InterpreterGroup. (It is recreated even after the InterpreterSetting is
closed)
* RemoteInterpreterManagedProcess. (It is not cleaned due to the connection
is not closed in PooledRemoteClient#remoteClientFactory)
* AuthorizeService#NoteAuth (it is not removed after note is removed)
### What type of PR is it?
[Bug Fix ]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-5254
### How should this be tested?
* Run large amount of cron jobs and check the instance count vi jmap
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jeff Zhang <[email protected]>
Closes #4056 from zjffdu/ZEPPELIN-5254 and squashes the following commits:
bf30be634 [Jeff Zhang] [ZEPPELIN-5254]. Memory leak of zeppelin server
---
.../zeppelin/interpreter/InterpreterSetting.java | 3 ++
.../interpreter/InterpreterSettingManager.java | 35 ++++++++++++----------
.../interpreter/ManagedInterpreterGroup.java | 3 ++
.../remote/ExecRemoteInterpreterProcess.java | 2 ++
.../remote/RemoteInterpreterProcess.java | 1 +
.../zeppelin/notebook/AuthorizationService.java | 4 +++
.../java/org/apache/zeppelin/notebook/Note.java | 3 ++
.../org/apache/zeppelin/notebook/Notebook.java | 1 +
8 files changed, 36 insertions(+), 16 deletions(-)
diff --git
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
index 0982cfc..e34fbd1 100644
---
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
+++
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
@@ -522,6 +522,9 @@ public class InterpreterSetting {
if (interpreterGroup != null) {
String sessionId = getInterpreterSessionId(executionContext);
interpreterGroup.close(sessionId);
+ if (interpreterGroup.isEmpty()) {
+ interpreterGroups.remove(interpreterGroup.getId());
+ }
}
}
diff --git
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
index 0a5241d..85c0955 100644
---
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
+++
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
@@ -737,18 +737,19 @@ public class InterpreterSettingManager implements
NoteEventListener, ClusterEven
r.getResourceId().getName());
}
} else if (remoteInterpreterProcess.isRunning()) {
- List<String> resourceList =
remoteInterpreterProcess.callRemoteFunction(client ->
client.resourcePoolGetAll());
- for (String res : resourceList) {
- resourceSet.add(Resource.fromJson(res));
- }
+ try {
+ List<String> resourceList =
remoteInterpreterProcess.callRemoteFunction(client ->
client.resourcePoolGetAll());
+ for (String res : resourceList) {
+ resourceSet.add(Resource.fromJson(res));
+ }
+
+ if (noteId != null) {
+ resourceSet = resourceSet.filterByNoteId(noteId);
+ }
+ if (paragraphId != null) {
+ resourceSet = resourceSet.filterByParagraphId(paragraphId);
+ }
- if (noteId != null) {
- resourceSet = resourceSet.filterByNoteId(noteId);
- }
- if (paragraphId != null) {
- resourceSet = resourceSet.filterByParagraphId(paragraphId);
- }
- try{
for (final Resource r : resourceSet) {
remoteInterpreterProcess.callRemoteFunction(client -> {
client.resourceRemove(r.getResourceId().getNoteId(),
@@ -757,7 +758,7 @@ public class InterpreterSettingManager implements
NoteEventListener, ClusterEven
return null;
});
}
- }catch (Exception e){
+ } catch (Exception e){
LOGGER.error(e.getMessage());
}
}
@@ -1110,10 +1111,12 @@ public class InterpreterSettingManager implements
NoteEventListener, ClusterEven
if (note.getParagraphs() != null) {
for (Paragraph paragraph : note.getParagraphs()) {
try {
- Interpreter interpreter = paragraph.getBindedInterpreter();
- InterpreterSetting interpreterSetting =
- ((ManagedInterpreterGroup)
interpreter.getInterpreterGroup()).getInterpreterSetting();
- restart(interpreterSetting.getId(), subject.getUser(), note.getId());
+ Interpreter interpreter = paragraph.getInterpreter();
+ if (interpreter != null) {
+ InterpreterSetting interpreterSetting =
+ ((ManagedInterpreterGroup)
interpreter.getInterpreterGroup()).getInterpreterSetting();
+ restart(interpreterSetting.getId(), subject.getUser(),
note.getId());
+ }
} catch (InterpreterNotFoundException e) {
} catch (InterpreterException e) {
diff --git
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java
index 3e6e23d..fb17542 100644
---
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java
+++
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java
@@ -183,4 +183,7 @@ public class ManagedInterpreterGroup extends
InterpreterGroup {
}
}
+ public boolean isEmpty() {
+ return this.sessions.isEmpty();
+ }
}
diff --git
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/ExecRemoteInterpreterProcess.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/ExecRemoteInterpreterProcess.java
index 1141513..824cfee 100644
---
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/ExecRemoteInterpreterProcess.java
+++
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/ExecRemoteInterpreterProcess.java
@@ -124,6 +124,8 @@ public class ExecRemoteInterpreterProcess extends
RemoteInterpreterManagedProces
this.interpreterProcessLauncher = null;
LOGGER.info("Remote exec process of interpreter group: {} is
terminated", getInterpreterGroupId());
} else {
+ // Shutdown connection
+ shutdown();
LOGGER.warn("Try to stop a not running interpreter process of
interpreter group: {}", getInterpreterGroupId());
}
}
diff --git
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
index 797e546..df81822 100644
---
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
+++
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
@@ -75,6 +75,7 @@ public abstract class RemoteInterpreterProcess implements
InterpreterClient {
public void shutdown() {
if (remoteClient != null) {
remoteClient.shutdown();
+ remoteClient = null;
}
}
diff --git
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/AuthorizationService.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/AuthorizationService.java
index 93797a7..28b263c 100644
---
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/AuthorizationService.java
+++
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/AuthorizationService.java
@@ -110,6 +110,10 @@ public class AuthorizationService implements
ClusterEventListener {
configStorage.save(new NotebookAuthorizationInfoSaving(this.notesAuth));
}
+ public void removeNoteAuth(String noteId) throws IOException {
+ this.notesAuth.remove(noteId);
+ }
+
// skip empty user and remove the white space around user name.
private Set<String> normalizeUsers(Set<String> users) {
Set<String> returnUser = new HashSet<>();
diff --git
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
index 10c9422..231b7bd 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
@@ -831,6 +831,9 @@ public class Note implements JsonSerializable {
.setStartTime(getStartTime())
.createExecutionContext();
setting.closeInterpreters(executionContext);
+ for (Paragraph p : paragraphs) {
+ p.setInterpreter(null);
+ }
}
}
}
diff --git
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
index 70f9dad..929e923 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
@@ -343,6 +343,7 @@ public class Notebook {
// Set Remove to true to cancel saving this note
note.setRemoved(true);
noteManager.removeNote(note.getId(), subject);
+ authorizationService.removeNoteAuth(note.getId());
fireNoteRemoveEvent(note, subject);
}
