Repository: tez Updated Branches: refs/heads/branch-0.6 bc56e10fe -> 3b61bee09
TEZ-2768. Log a useful error message when the summary stream cannot be closed when shutting down an AM. (Jeff Zhang via hitesh) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/3b61bee0 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/3b61bee0 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/3b61bee0 Branch: refs/heads/branch-0.6 Commit: 3b61bee09f22653c4a28a8ca23cda470652b57bf Parents: bc56e10 Author: Hitesh Shah <[email protected]> Authored: Fri Sep 4 16:20:03 2015 -0700 Committer: Hitesh Shah <[email protected]> Committed: Fri Sep 4 16:20:03 2015 -0700 ---------------------------------------------------------------------- CHANGES.txt | 4 ++++ .../tez/dag/api/client/TimelineReaderFactory.java | 7 ++++--- .../tez/dag/history/recovery/RecoveryService.java | 18 ++++++++++++++++-- 3 files changed, 24 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/3b61bee0/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 8ef031f..48c804c 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -6,6 +6,8 @@ Release 0.6.3: Unreleased INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-2768. Log a useful error message when the summary stream cannot be closed when shutting + down an AM. TEZ-2745. ClassNotFoundException of user code should fail dag TEZ-2752. logUnsuccessful completion in Attempt should write original finish time to ATS @@ -233,6 +235,8 @@ INCOMPATIBLE CHANGES TEZ-2552. CRC errors can cause job to run for very long time in large jobs. ALL CHANGES: + TEZ-2768. Log a useful error message when the summary stream cannot be closed when shutting + down an AM. TEZ-2745. ClassNotFoundException of user code should fail dag TEZ-2752. logUnsuccessful completion in Attempt should write original finish time to ATS http://git-wip-us.apache.org/repos/asf/tez/blob/3b61bee0/tez-api/src/main/java/org/apache/tez/dag/api/client/TimelineReaderFactory.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/TimelineReaderFactory.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/TimelineReaderFactory.java index c0569dd..4a8e172 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/TimelineReaderFactory.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/TimelineReaderFactory.java @@ -37,6 +37,9 @@ import com.sun.jersey.api.client.config.DefaultClientConfig; import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory; import com.sun.jersey.client.urlconnection.URLConnectionClientHandler; import com.sun.jersey.json.impl.provider.entity.JSONRootElementProvider; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; @@ -46,8 +49,6 @@ import org.apache.hadoop.security.authentication.client.ConnectionConfigurator; import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.tez.common.ReflectionUtils; import org.apache.tez.dag.api.TezException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /* * TimelineReaderFactory getTimelineReaderStrategy returns a Strategy class, which is used to @@ -63,7 +64,7 @@ import org.slf4j.LoggerFactory; @InterfaceAudience.Private public class TimelineReaderFactory { - private static final Logger LOG = LoggerFactory.getLogger(TimelineReaderFactory.class); + private static final Log LOG = LogFactory.getLog(TimelineReaderFactory.class); private static final String KERBEROS_DELEGATION_TOKEN_AUTHENTICATOR_CLAZZ_NAME = "org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator"; http://git-wip-us.apache.org/repos/asf/tez/blob/3b61bee0/tez-dag/src/main/java/org/apache/tez/dag/history/recovery/RecoveryService.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/recovery/RecoveryService.java b/tez-dag/src/main/java/org/apache/tez/dag/history/recovery/RecoveryService.java index c0788dd..8a07211 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/recovery/RecoveryService.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/recovery/RecoveryService.java @@ -215,7 +215,13 @@ public class RecoveryService extends AbstractService { summaryStream.hflush(); summaryStream.close(); } catch (IOException ioe) { - LOG.warn("Error when closing summary stream", ioe); + if (!recoveryDirFS.exists(recoveryPath)) { + LOG.warn("Ignoring error while closing summary stream." + + " The recovery directory at " + recoveryPath + + " has already been deleted externally"); + } else { + LOG.warn("Error when closing summary stream", ioe); + } } } for (Entry<TezDAGID, FSDataOutputStream> entry : outputStreamMap.entrySet()) { @@ -224,7 +230,15 @@ public class RecoveryService extends AbstractService { entry.getValue().hflush(); entry.getValue().close(); } catch (IOException ioe) { - LOG.warn("Error when closing output stream", ioe); + if (!recoveryDirFS.exists(recoveryPath)) { + LOG.warn("Ignoring error while closing output stream." + + " The recovery directory at " + recoveryPath + + " has already been deleted externally"); + // avoid closing other outputStream as the recovery directory has already been deleted. + break; + } else { + LOG.warn("Error when closing output stream", ioe); + } } } }
