Author: liyin Date: Tue May 7 18:31:38 2013 New Revision: 1480005 URL: http://svn.apache.org/r1480005 Log: [HBASE-8484] Adding diagnostic messages when the RegionServer hists an IOException in RegionServerReport
Author: manukranthk Summary: This diff adds instrumentation in the case when we catch IOException in regionserver report. Test Plan: Tested on Dev Cluster by killing the master. Reviewers: aaiyer Reviewed By: aaiyer Differential Revision: https://phabricator.fb.com/D797516 Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1480005&r1=1480004&r2=1480005&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original) +++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Tue May 7 18:31:38 2013 @@ -640,6 +640,43 @@ public class HRegionServer implements HR return false; } + private final static String NEWLINE = System.getProperty("line.separator"); + public static String printFailedRegionserverReport(HServerInfo server, + HMsg[] inMsgs, HRegionInfo[] loadedRegions, HMsg[] outMsgs, Throwable t) { + StringBuilder sb = new StringBuilder(); + sb.append("Could not send the response to the regionserver. " + + "Printing diagnostic messages"); + sb.append(NEWLINE); + sb.append("ServerAddress : " + server.getHostnamePort()); + sb.append(NEWLINE); + sb.append("Incomming Messages :"); + sb.append(NEWLINE); + if (inMsgs != null) { + for (HMsg msg : inMsgs) { + sb.append("HMsg : " + msg.toString()); + sb.append(NEWLINE); + } + } + if (loadedRegions != null) { + sb.append("Loaded Regions : "); + for (HRegionInfo info : loadedRegions) { + sb.append(info.getRegionNameAsString()); + sb.append(NEWLINE); + } + } + sb.append("Outgoing Messages :"); + sb.append(NEWLINE); + if (outMsgs != null) { + for (HMsg msg : outMsgs) { + sb.append("HMsg : " + msg.toString()); + sb.append(NEWLINE); + } + } + sb.append("Related Exception : " + t); + String s = sb.toString(); + return s; + } + /** * The HRegionServer sticks in this loop until closed. It repeatedly checks * in with the HMaster, sending heartbeats & reports, and receiving HRegion @@ -696,9 +733,19 @@ public class HRegionServer implements HR // XXX add a field in serverInfo to report to fsOK to master? this.serverInfo.setLoad(hsl); addOutboundMsgs(outboundMessages); - HMsg msgs[] = this.hbaseMaster.regionServerReport( - serverInfo, outboundMessages.toArray(EMPTY_HMSG_ARRAY), - getMostLoadedRegions()); + HMsg msgs[] = null; + LOG.debug("Attempting regionserver report with the master"); + try { + msgs = this.hbaseMaster.regionServerReport( + serverInfo, outboundMessages.toArray(EMPTY_HMSG_ARRAY), + getMostLoadedRegions()); + } catch (IOException e) { + LOG.debug("RegionServerReport failed."); + LOG.debug(HRegionServer.printFailedRegionserverReport(this.serverInfo, + outboundMessages.toArray(EMPTY_HMSG_ARRAY), + getMostLoadedRegions(), msgs, (Throwable)e)); + } + LOG.debug("Attempted regionserver report with the master"); lastMsg = System.currentTimeMillis(); updateOutboundMsgs(outboundMessages); outboundMessages.clear(); @@ -712,7 +759,8 @@ public class HRegionServer implements HR continue; } - if (InjectionHandler.falseCondition(InjectionEvent.HREGIONSERVER_REPORT_RESPONSE, (Object[])msgs)) { + if (InjectionHandler.falseCondition( + InjectionEvent.HREGIONSERVER_REPORT_RESPONSE, (Object[])msgs)) { continue; }
