chia7712 commented on a change in pull request #9262:
URL: https://github.com/apache/kafka/pull/9262#discussion_r489476268
##########
File path:
streams/src/main/java/org/apache/kafka/streams/processor/internals/StateDirectory.java
##########
@@ -306,40 +306,55 @@ public synchronized void clean() {
*/
public synchronized void cleanRemovedTasks(final long cleanupDelayMs) {
try {
- cleanRemovedTasks(cleanupDelayMs, false);
+ cleanRemovedTasksCalledByCleanerThread(cleanupDelayMs);
} catch (final Exception cannotHappen) {
throw new IllegalStateException("Should have swallowed
exception.", cannotHappen);
}
}
- private synchronized void cleanRemovedTasks(final long cleanupDelayMs,
- final boolean manualUserCall)
throws Exception {
- final File[] taskDirs = listAllTaskDirectories();
- if (taskDirs == null || taskDirs.length == 0) {
- return; // nothing to do
- }
-
- for (final File taskDir : taskDirs) {
+ private void cleanRemovedTasksCalledByCleanerThread(final long
cleanupDelayMs) {
+ for (final File taskDir : listAllTaskDirectories()) {
final String dirName = taskDir.getName();
final TaskId id = TaskId.parse(dirName);
if (!locks.containsKey(id)) {
- Exception exception = null;
try {
if (lock(id)) {
final long now = time.milliseconds();
final long lastModifiedMs = taskDir.lastModified();
if (now > lastModifiedMs + cleanupDelayMs) {
log.info("{} Deleting obsolete state directory {}
for task {} as {}ms has elapsed (cleanup delay is {}ms).",
logPrefix(), dirName, id, now -
lastModifiedMs, cleanupDelayMs);
-
- Utils.delete(taskDir,
Collections.singletonList(new File(taskDir, LOCK_FILE_NAME)));
- } else if (manualUserCall) {
- log.info("{} Deleting state directory {} for task
{} as user calling cleanup.",
- logPrefix(), dirName, id);
-
Utils.delete(taskDir,
Collections.singletonList(new File(taskDir, LOCK_FILE_NAME)));
}
}
+ } catch (final OverlappingFileLockException | IOException e) {
+ log.warn("{} Swallowed the following exception during
deletion of obsolete state directory {} for task {}: {}",
+ logPrefix(), dirName, id, e);
+ } finally {
+ try {
+ unlock(id);
+ } catch (final IOException e) {
+ log.warn("{} Swallowed the following exception during
unlocking after " +
+ "deletion of obsolete state directory for task
{}: {}",
+ logPrefix(), dirName, e);
+ }
+ }
+ }
+ }
+ }
+
+ private void cleanRemovedTasksCalledByUser() throws Exception {
+ for (final File taskDir : listAllTaskDirectories()) {
+ final String dirName = taskDir.getName();
+ final TaskId id = TaskId.parse(dirName);
+ if (!locks.containsKey(id)) {
+ Exception exception = null;
+ try {
+ if (lock(id)) {
+ log.info("{} Deleting state directory {} for task {}
as user calling cleanup.",
+ logPrefix(), dirName, id);
+ Utils.delete(taskDir, Collections.singletonList(new
File(taskDir, LOCK_FILE_NAME)));
+ }
} catch (final OverlappingFileLockException | IOException e) {
Review comment:
the method ```clean``` catches ```Exception``` already. Could we get rid
of those try-catch statements? the code ```log.error("{} Failed to release the
state directory lock.", logPrefix());``` can be moved to ```clean```. For
example:
```java
public synchronized void clean() {
// remove task dirs
try {
cleanRemovedTasksCalledByUser();
} catch (final Exception e) {
log.error("{} Failed to release the state directory lock.",
logPrefix());
throw new StreamsException(e);
}
```
```java
private void cleanRemovedTasksCalledByUser() throws Exception {
for (final File taskDir : listAllTaskDirectories()) {
final String dirName = taskDir.getName();
final TaskId id = TaskId.parse(dirName);
if (!locks.containsKey(id) && lock(id)) {
try {
log.info("{} Deleting state directory {} for task {} as
user calling cleanup.",
logPrefix(), dirName, id);
Utils.delete(taskDir, Collections.singletonList(new
File(taskDir, LOCK_FILE_NAME)));
} finally {
unlock(id);
// for manual user call, stream threads are not running
so it is safe to delete
// the whole directory
Utils.delete(taskDir);
}
```
##########
File path:
streams/src/main/java/org/apache/kafka/streams/processor/internals/StateDirectory.java
##########
@@ -280,7 +280,7 @@ synchronized void unlock(final TaskId taskId) throws
IOException {
public synchronized void clean() {
// remove task dirs
try {
- cleanRemovedTasks(0, true);
+ cleanRemovedTasksCalledByUser();
} catch (final Exception e) {
// this is already logged within cleanRemovedTasks
Review comment:
cleanRemovedTasks -> cleanRemovedTasksCalledByUser
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
