This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push: new 4917b3032d Fixes #3031 Makes scans more tolerant of Hadoop sutdown hook. (#3032) 4917b3032d is described below commit 4917b3032deb097adf1573086c0c3c3f11265f0a Author: Keith Turner <ktur...@apache.org> AuthorDate: Wed Oct 19 20:53:16 2022 +0100 Fixes #3031 Makes scans more tolerant of Hadoop sutdown hook. (#3032) --- .../apache/accumulo/core/iterators/Combiner.java | 3 ++- .../apache/accumulo/core/util/ShutdownUtil.java | 24 ++++++++++++++++++++++ .../main/java/org/apache/accumulo/fate/Fate.java | 15 +------------- .../apache/accumulo/tserver/tablet/Scanner.java | 11 +++++++--- .../apache/accumulo/tserver/tablet/TabletBase.java | 11 +++++++++- 5 files changed, 45 insertions(+), 19 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java b/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java index 23bbfeade9..fedb25ca90 100644 --- a/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java +++ b/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java @@ -21,6 +21,7 @@ package org.apache.accumulo.core.iterators; import static java.util.concurrent.TimeUnit.HOURS; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.Collection; import java.util.Iterator; import java.util.List; @@ -130,7 +131,7 @@ public abstract class Combiner extends WrappingIterator implements OptionDescrib source.next(); hasNext = _hasNext(); } catch (IOException e) { - throw new RuntimeException(e); + throw new UncheckedIOException(e); } return topValue; } diff --git a/core/src/main/java/org/apache/accumulo/core/util/ShutdownUtil.java b/core/src/main/java/org/apache/accumulo/core/util/ShutdownUtil.java index b1cd2f0afb..0fbf90fbda 100644 --- a/core/src/main/java/org/apache/accumulo/core/util/ShutdownUtil.java +++ b/core/src/main/java/org/apache/accumulo/core/util/ShutdownUtil.java @@ -18,6 +18,8 @@ */ package org.apache.accumulo.core.util; +import java.io.IOException; + public class ShutdownUtil { /** @@ -34,4 +36,26 @@ public class ShutdownUtil { return false; } + public static boolean isIOException(Throwable e) { + if (e == null) + return false; + + if (e instanceof IOException) + return true; + + for (Throwable suppressed : e.getSuppressed()) + if (isIOException(suppressed)) + return true; + + return isIOException(e.getCause()); + } + + /** + * @return true if there is a possibility that the exception was caused by the hadoop shutdown + * hook closing the hadoop file system objects, otherwise false + */ + public static boolean wasCausedByHadoopShutdown(Exception e) { + return isShutdownInProgress() && isIOException(e); + } + } diff --git a/core/src/main/java/org/apache/accumulo/fate/Fate.java b/core/src/main/java/org/apache/accumulo/fate/Fate.java index d45f039e28..2a98b6e15b 100644 --- a/core/src/main/java/org/apache/accumulo/fate/Fate.java +++ b/core/src/main/java/org/apache/accumulo/fate/Fate.java @@ -20,6 +20,7 @@ package org.apache.accumulo.fate; import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.accumulo.core.util.ShutdownUtil.isIOException; import static org.apache.accumulo.fate.ReadOnlyTStore.TStatus.FAILED; import static org.apache.accumulo.fate.ReadOnlyTStore.TStatus.FAILED_IN_PROGRESS; import static org.apache.accumulo.fate.ReadOnlyTStore.TStatus.IN_PROGRESS; @@ -135,20 +136,6 @@ public class Fate<T> { } } - private boolean isIOException(Throwable e) { - if (e == null) - return false; - - if (e instanceof IOException) - return true; - - for (Throwable suppressed : e.getSuppressed()) - if (isIOException(suppressed)) - return true; - - return isIOException(e.getCause()); - } - /** * The Hadoop Filesystem registers a java shutdown hook that closes the file system. This can * cause threads to get spurious IOException. If this happens, instead of failing a FATE diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Scanner.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Scanner.java index 08e2f20afa..2fd917ecda 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Scanner.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Scanner.java @@ -125,16 +125,21 @@ public class Scanner { else throw iie; } catch (IOException ioe) { - if (ShutdownUtil.isShutdownInProgress()) { + if (ShutdownUtil.wasCausedByHadoopShutdown(ioe)) { log.debug("IOException while shutdown in progress ", ioe); - throw new TabletClosedException(ioe); // assume IOException was caused by execution of HDFS - // shutdown hook + throw new TabletClosedException(ioe); // this was possibly caused by Hadoop shutdown hook, + // so make the client retry } sawException = true; dataSource.close(true); throw ioe; } catch (RuntimeException re) { + if (ShutdownUtil.wasCausedByHadoopShutdown(re)) { + log.debug("RuntimeException while shutdown in progress ", re); + throw new TabletClosedException(re); // this was possibly caused by Hadoop shutdown hook, so + // make the client retry + } sawException = true; throw re; } finally { diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java index 031adc0619..ce8cddc348 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java @@ -410,7 +410,7 @@ public abstract class TabletBase { entriesAdded); tabletClosed = true; } catch (IOException ioe) { - if (ShutdownUtil.isShutdownInProgress()) { + if (ShutdownUtil.wasCausedByHadoopShutdown(ioe)) { // assume HDFS shutdown hook caused this exception log.debug("IOException while shutdown in progress", ioe); handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, @@ -431,6 +431,15 @@ public abstract class TabletBase { handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded); tabletClosed = true; + } catch (RuntimeException re) { + if (ShutdownUtil.wasCausedByHadoopShutdown(re)) { + log.debug("RuntimeException while shutdown in progress", re); + handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, + entriesAdded); + tabletClosed = true; + } else { + throw re; + } } }