Author: stack
Date: Fri Jul 10 19:52:20 2009
New Revision: 793094
URL: http://svn.apache.org/viewvc?rev=793094&view=rev
Log:
HBASE-1603 MR failed 'RetriesExhaustedException: Trying to contact region
server Some server for region TestTable...'
Modified:
hadoop/hbase/trunk/CHANGES.txt
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
Modified: hadoop/hbase/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=793094&r1=793093&r2=793094&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Fri Jul 10 19:52:20 2009
@@ -463,6 +463,8 @@
HBASE-1640 Allow passing arguments to jruby script run when run by
bin/hbase shell
HBASE-698 HLog recovery is not performed after master failure
HBASE-1643 ScanDeleteTracker takes comparator but it unused
+ HBASE-1603 MR failed "RetriesExhaustedException: Trying to contact region
server
+ Some server for region TestTable..." -- deubugging
OPTIMIZATIONS
HBASE-1412 Change values for delete column and column family in KeyValue
Modified:
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=793094&r1=793093&r2=793094&view=diff
==============================================================================
---
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
(original)
+++
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
Fri Jul 10 19:52:20 2009
@@ -606,8 +606,7 @@
// instantiate the location
location = new HRegionLocation(regionInfo,
- new HServerAddress(serverAddress));
- LOG.debug(location);
+ new HServerAddress(serverAddress));
cacheLocation(tableName, location);
return location;
} catch (TableNotFoundException e) {
@@ -744,7 +743,7 @@
if (rl != null && LOG.isDebugEnabled()) {
LOG.debug("Removed " +
rl.getRegionInfo().getRegionNameAsString() +
" for tableName=" + Bytes.toString(tableName) + " from cache "
+
- "because of " + Bytes.toString(row));
+ "because of " + Bytes.toStringBinary(row));
}
}
}
@@ -780,7 +779,9 @@
byte [] startKey = location.getRegionInfo().getStartKey();
SoftValueSortedMap<byte [], HRegionLocation> tableLocations =
getTableLocations(tableName);
- tableLocations.put(startKey, location);
+ if (tableLocations.put(startKey, location) == null) {
+ LOG.debug("Cached location " + location);
+ }
}
public HRegionInterface getHRegionConnection(
@@ -973,7 +974,7 @@
List<Throwable> exceptions = new ArrayList<Throwable>();
HRegionLocation location = null;
int tries = 0;
- while (tries < numRetries) {
+ for (; tries < numRetries;) {
try {
location = getRegionLocation(tableName, rowKey, reloadFlag);
} catch (Throwable t) {
@@ -1003,7 +1004,6 @@
return;
}
boolean retryOnlyOne = false;
- int tries = 0;
if (list.size() > 1) {
Collections.sort(list);
}
@@ -1014,37 +1014,43 @@
byte [] currentRegion = location.getRegionInfo().getRegionName();
byte [] region = currentRegion;
boolean isLastRow = false;
- for (int i = 0; i < list.size() && tries < numRetries; i++) {
+ Put [] putarray = new Put[0];
+ for (int i = 0, tries = 0; i < list.size() && tries < this.numRetries;
i++) {
Put put = list.get(i);
currentPuts.add(put);
+ // If the next Put goes to a new region, then we are to clear
+ // currentPuts now during this cycle.
isLastRow = (i + 1) == list.size();
if (!isLastRow) {
location = getRegionLocationForRowWithRetries(tableName,
- list.get(i+1).getRow(), false);
+ list.get(i + 1).getRow(), false);
region = location.getRegionInfo().getRegionName();
}
if (!Bytes.equals(currentRegion, region) || isLastRow || retryOnlyOne)
{
- final Put [] puts = currentPuts.toArray(new Put[0]);
+ final Put [] puts = currentPuts.toArray(putarray);
int index = getRegionServerWithRetries(new ServerCallable<Integer>(
this, tableName, put.getRow()) {
public Integer call() throws IOException {
- int i = server.put(location.getRegionInfo()
- .getRegionName(), puts);
- return i;
+ return server.put(location.getRegionInfo().getRegionName(),
puts);
}
});
+ // index is == -1 if all puts processed successfully, else its index
+ // of last Put successfully processed.
if (index != -1) {
if (tries == numRetries - 1) {
- throw new RetriesExhaustedException("Some server",
- currentRegion, put.getRow(),
- tries, new ArrayList<Throwable>());
+ throw new RetriesExhaustedException("Some server, retryOnlyOne="
+
+ retryOnlyOne + ", index=" + index + ", islastrow=" + isLastRow
+
+ ", tries=" + tries + ", numtries=" + numRetries + ", i=" + i +
+ ", listsize=" + list.size() + ", location=" + location +
+ ", region=" + Bytes.toStringBinary(region),
+ currentRegion, put.getRow(), tries, new
ArrayList<Throwable>());
}
long sleepTime = getPauseTime(tries);
if (LOG.isDebugEnabled()) {
- LOG.debug("Reloading region " + Bytes.toString(currentRegion) +
+ LOG.debug("Reloading region " +
Bytes.toStringBinary(currentRegion) +
" location because regionserver didn't accept updates; " +
- "tries=" + tries +
- " of max=" + this.numRetries + ", waiting=" + sleepTime +
"ms");
+ "tries=" + tries + " of max=" + this.numRetries +
+ ", waiting=" + sleepTime + "ms");
}
try {
Thread.sleep(sleepTime);
@@ -1054,12 +1060,14 @@
}
i = i - puts.length + index;
retryOnlyOne = true;
+ // Reload location.
location = getRegionLocationForRowWithRetries(tableName,
list.get(i + 1).getRow(), true);
region = location.getRegionInfo().getRegionName();
- }
- else {
+ } else {
+ // Reset these flags/counters on successful batch Put
retryOnlyOne = false;
+ tries = 0;
}
currentRegion = region;
currentPuts.clear();
Modified:
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java?rev=793094&r1=793093&r2=793094&view=diff
==============================================================================
---
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java
(original)
+++
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java
Fri Jul 10 19:52:20 2009
@@ -47,9 +47,9 @@
StringBuilder buffer = new StringBuilder("Trying to contact region server
");
buffer.append(serverName);
buffer.append(" for region ");
- buffer.append(regionName == null? "": Bytes.toString(regionName));
+ buffer.append(regionName == null? "": Bytes.toStringBinary(regionName));
buffer.append(", row '");
- buffer.append(row == null? "": Bytes.toString(row));
+ buffer.append(row == null? "": Bytes.toStringBinary(row));
buffer.append("', but failed after ");
buffer.append(numTries + 1);
buffer.append(" attempts.\nExceptions:\n");
@@ -59,4 +59,4 @@
}
return buffer.toString();
}
-}
\ No newline at end of file
+}
Modified:
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java?rev=793094&r1=793093&r2=793094&view=diff
==============================================================================
---
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
(original)
+++
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
Fri Jul 10 19:52:20 2009
@@ -102,7 +102,8 @@
*
* @param regionName
* @param puts
- * @return The number of processed put's.
+ * @return The number of processed put's. Returns -1 if all Puts
+ * processed successfully.
* @throws IOException
*/
public int put(final byte[] regionName, final Put [] puts)
Modified:
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=793094&r1=793093&r2=793094&view=diff
==============================================================================
---
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
(original)
+++
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
Fri Jul 10 19:52:20 2009
@@ -941,7 +941,7 @@
// all and sundry.
this.log.abortCacheFlush();
DroppedSnapshotException dse = new DroppedSnapshotException("region: " +
- Bytes.toString(getRegionName()));
+ Bytes.toStringBinary(getRegionName()));
dse.initCause(t);
throw dse;
}
@@ -1383,7 +1383,7 @@
requestFlush();
if (!blocked) {
LOG.info("Blocking updates for '" + Thread.currentThread().getName() +
- "' on region " + Bytes.toString(getRegionName()) +
+ "' on region " + Bytes.toStringBinary(getRegionName()) +
": memstore size " +
StringUtils.humanReadableInt(this.memstoreSize.get()) +
" is >= than blocking " +
@@ -1521,9 +1521,9 @@
if(!rowIsInRange(regionInfo, row)) {
throw new WrongRegionException("Requested row out of range for " +
"HRegion " + this + ", startKey='" +
- Bytes.toString(regionInfo.getStartKey()) + "', getEndKey()='" +
- Bytes.toString(regionInfo.getEndKey()) + "', row='" +
- Bytes.toString(row) + "'");
+ Bytes.toStringBinary(regionInfo.getStartKey()) + "', getEndKey()='" +
+ Bytes.toStringBinary(regionInfo.getEndKey()) + "', row='" +
+ Bytes.toStringBinary(row) + "'");
}
}
Modified:
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=793094&r1=793093&r2=793094&view=diff
==============================================================================
---
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
(original)
+++
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
Fri Jul 10 19:52:20 2009
@@ -1772,6 +1772,7 @@
public int put(final byte[] regionName, final Put [] puts)
throws IOException {
+ // Count of Puts processed.
int i = 0;
checkOpen();
try {
@@ -1783,13 +1784,15 @@
locks[i] = getLockFromId(puts[i].getLockId());
region.put(puts[i], locks[i]);
}
- } catch(WrongRegionException ex) {
+ } catch (WrongRegionException ex) {
+ LOG.debug("Batch puts: " + i, ex);
return i;
} catch (NotServingRegionException ex) {
return i;
} catch (Throwable t) {
throw convertThrowableToIOE(cleanup(t));
}
+ // All have been processed successfully.
return -1;
}
