Author: stack
Date: Fri Dec 17 05:39:22 2010
New Revision: 1050277
URL: http://svn.apache.org/viewvc?rev=1050277&view=rev
Log:
HBASE-3362 If .META. offline between OPENING and OPENED, then wrong server
location in .META. is possible
Modified:
hbase/branches/0.90/CHANGES.txt
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
Modified: hbase/branches/0.90/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hbase/branches/0.90/CHANGES.txt?rev=1050277&r1=1050276&r2=1050277&view=diff
==============================================================================
--- hbase/branches/0.90/CHANGES.txt (original)
+++ hbase/branches/0.90/CHANGES.txt Fri Dec 17 05:39:22 2010
@@ -752,6 +752,8 @@ Release 0.90.0 - Unreleased
HBASE-3360 ReplicationLogCleaner is enabled by default in 0.90 -- causes
NPE
HBASE-3363 ReplicationSink should batch delete
HBASE-3365 EOFE contacting crashed RS causes Master abort
+ HBASE-3362 If .META. offline between OPENING and OPENED, then wrong server
+ location in .META. is possible
IMPROVEMENTS
Modified:
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
URL:
http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java?rev=1050277&r1=1050276&r2=1050277&view=diff
==============================================================================
---
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
(original)
+++
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
Fri Dec 17 05:39:22 2010
@@ -20,7 +20,7 @@
package org.apache.hadoop.hbase.regionserver.handler;
import java.io.IOException;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -32,7 +32,6 @@ import org.apache.hadoop.hbase.regionser
import org.apache.hadoop.hbase.zookeeper.ZKAssign;
import org.apache.hadoop.util.Progressable;
import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.KeeperException.Code;
/**
* Handles opening of a region on a region server.
@@ -46,6 +45,11 @@ public class OpenRegionHandler extends E
private final HRegionInfo regionInfo;
+ // We get version of our znode at start of open process and monitor it across
+ // the total open. We'll fail the open if someone hijacks our znode; we can
+ // tell this has happened if version is not as expected.
+ private volatile int version = -1;
+
public OpenRegionHandler(final Server server,
final RegionServerServices rsServices, HRegionInfo regionInfo) {
this(server, rsServices, regionInfo, EventType.M_RS_OPEN_REGION);
@@ -73,126 +77,192 @@ public class OpenRegionHandler extends E
}
final String encodedName = regionInfo.getEncodedName();
- // TODO: Previously we would check for root region availability (but only
that it
- // was initially available, does not check if it later went away)
- // Do we need to wait on both root and meta to be available to open a
region
- // now since we edit meta?
-
// Check that this region is not already online
HRegion region = this.rsServices.getFromOnlineRegions(encodedName);
if (region != null) {
- LOG.warn("Attempting open of " + name +
- " but it's already online on this server");
+ LOG.warn("Attempted open of " + name +
+ " but already online on this server");
return;
}
- int openingVersion = transitionZookeeperOfflineToOpening(encodedName);
- if (openingVersion == -1) return;
-
- // Open the region
- final AtomicInteger openingInteger = new AtomicInteger(openingVersion);
- try {
- // Instantiate the region. This also periodically updates OPENING.
- region = HRegion.openHRegion(regionInfo, this.rsServices.getWAL(),
- server.getConfiguration(), this.rsServices.getFlushRequester(),
- new Progressable() {
- public void progress() {
- try {
- int vsn = ZKAssign.retransitionNodeOpening(
- server.getZooKeeper(), regionInfo, server.getServerName(),
- openingInteger.get());
- if (vsn == -1) {
- throw KeeperException.create(Code.BADVERSION);
- }
- openingInteger.set(vsn);
- } catch (KeeperException e) {
- server.abort("ZK exception refreshing OPENING node; " + name, e);
- }
- }
- });
- } catch (IOException e) {
- LOG.error("Failed open of " + regionInfo +
- "; resetting state of transition node from OPENING to OFFLINE", e);
- try {
- // TODO: We should rely on the master timing out OPENING instead of
this
- // TODO: What if this was a split open? The RS made the OFFLINE
- // znode, not the master.
- ZKAssign.forceNodeOffline(server.getZooKeeper(), regionInfo,
- server.getServerName());
- } catch (KeeperException e1) {
- LOG.error("Error forcing node back to OFFLINE from OPENING; " + name);
- }
- return;
+ // If fails, just return. Someone stole the region from under us.
+ // Calling transitionZookeeperOfflineToOpening initalizes this.version.
+ if (!transitionZookeeperOfflineToOpening(encodedName)) return;
+
+ // Open region. After a successful open, failures in subsequent processing
+ // needs to do a close as part of cleanup.
+ region = openRegion();
+ if (region == null) return;
+ boolean failed = true;
+ if (tickleOpening("post_region_open")) {
+ if (updateMeta(region)) failed = false;
}
- // Region is now open. Close it if error.
- // Re-transition node to OPENING again to verify no one has stomped on us
- openingVersion = openingInteger.get();
- try {
- if ((openingVersion = ZKAssign.retransitionNodeOpening(
- server.getZooKeeper(), regionInfo, server.getServerName(),
- openingVersion)) == -1) {
- LOG.warn("Completed the OPEN of region " + name +
- " but when transitioning from " +
- " OPENING to OPENING got a version mismatch, someone else clashed " +
- "-- closing region");
- cleanupFailedOpen(region);
- return;
- }
- } catch (KeeperException e) {
- LOG.error("Failed transitioning node " + name +
- " from OPENING to OPENED -- closing region", e);
- cleanupFailedOpen(region);
- return;
- } catch (IOException e) {
- LOG.error("Failed to close region " + name +
- " after failing to transition -- closing region", e);
+ if (failed) {
cleanupFailedOpen(region);
return;
}
- // Update ZK, ROOT or META
- try {
- this.rsServices.postOpenDeployTasks(region,
- this.server.getCatalogTracker(), false);
- } catch (IOException e) {
- LOG.error("Error updating " + name + " location in catalog table -- " +
- "closing region", e);
- cleanupFailedOpen(region);
- return;
- } catch (KeeperException e) {
- // TODO: rollback the open?
- LOG.error("ZK Error updating " + name + " location in catalog " +
- "table -- closing region", e);
+ if (!transitionToOpened(region)) {
cleanupFailedOpen(region);
return;
}
+ // Done! Successful region open
+ LOG.debug("Opened " + name);
+ }
+
+ /**
+ * Update ZK, ROOT or META. This can take a while if for example the
+ * .META. is not available -- if server hosting .META. crashed and we are
+ * waiting on it to come back -- so run in a thread and keep updating znode
+ * state meantime so master doesn't timeout our region-in-transition.
+ * Caller must cleanup region if this fails.
+ */
+ private boolean updateMeta(final HRegion r) {
+ // Object we do wait/notify on. Make it boolean. If set, we're done.
+ // Else, wait.
+ final AtomicBoolean signaller = new AtomicBoolean(false);
+ PostOpenDeployTasksThread t = new PostOpenDeployTasksThread(r,
+ this.server, this.rsServices, signaller);
+ t.start();
+ int assignmentTimeout = this.server.getConfiguration().
+ getInt("hbase.master.assignment.timeoutmonitor.period", 10000);
+ // Total timeout for meta edit. If we fail adding the edit then close out
+ // the region and let it be assigned elsewhere.
+ long timeout = assignmentTimeout * 10;
+ long now = System.currentTimeMillis();
+ long endTime = now + timeout;
+ // Let our period at which we update OPENING state to be be 1/3rd of the
+ // regions-in-transition timeout period.
+ long period = Math.max(1, assignmentTimeout/ 3);
+ long lastUpdate = now;
+ while (!signaller.get() && t.isAlive() && !this.server.isStopped() &&
+ !this.rsServices.isStopping() && (endTime > now)) {
+ long elapsed = now - lastUpdate;
+ if (elapsed > period) {
+ // Only tickle OPENING if postOpenDeployTasks is taking some time.
+ lastUpdate = now;
+ tickleOpening("post_open_deploy");
+ }
+ synchronized (signaller) {
+ try {
+ signaller.wait(period);
+ } catch (InterruptedException e) {
+ // Go to the loop check.
+ }
+ }
+ now = System.currentTimeMillis();
+ }
+ // Is thread still alive? We may have left above loop because server is
+ // stopping or we timed out the edit. Is so, interrupt it.
+ if (t.isAlive()) {
+ LOG.debug("Interrupting thread " + t);
+ t.interrupt();
+ }
+ // Was there an exception opening the region? This should trigger on
+ // InterruptedException too. If so, we failed.
+ return t.getException() == null;
+ }
+
+ /**
+ * Thread to run region post open tasks. Call {...@link #getException()}
after
+ * the thread finishes to check for exceptions running
+ * {...@link RegionServerServices#postOpenDeployTasks(HRegion,
org.apache.hadoop.hbase.catalog.CatalogTracker, boolean)}.
+ */
+ static class PostOpenDeployTasksThread extends Thread {
+ private Exception exception = null;
+ private final Server server;
+ private final RegionServerServices services;
+ private final HRegion region;
+ private final AtomicBoolean signaller;
+
+ PostOpenDeployTasksThread(final HRegion region, final Server server,
+ final RegionServerServices services, final AtomicBoolean signaller) {
+ super("PostOpenDeployTasks:" + region.getRegionInfo().getEncodedName());
+ this.setDaemon(true);
+ this.server = server;
+ this.services = services;
+ this.region = region;
+ this.signaller = signaller;
+ }
+
+ public void run() {
+ try {
+ this.services.postOpenDeployTasks(this.region,
+ this.server.getCatalogTracker(), false);
+ } catch (Exception e) {
+ LOG.warn("Exception running postOpenDeployTasks; region=" +
+ this.region.getRegionInfo().getEncodedName(), e);
+ this.exception = e;
+ }
+ // We're done. Set flag then wake up anyone waiting on thread to
complete.
+ this.signaller.set(true);
+ synchronized (this.signaller) {
+ this.signaller.notify();
+ }
+ }
+
+ /**
+ * @return Null or the run exception; call this method after thread is
done.
+ */
+ Exception getException() {
+ return this.exception;
+ }
+ }
+
+ /**
+ * @param r Region we're working on.
+ * @return Transition znode to OPENED state.
+ * @throws IOException
+ */
+ private boolean transitionToOpened(final HRegion r) throws IOException {
+ boolean result = false;
+ HRegionInfo hri = r.getRegionInfo();
+ final String name = hri.getRegionNameAsString();
// Finally, Transition ZK node to OPENED
try {
- if (ZKAssign.transitionNodeOpened(server.getZooKeeper(), regionInfo,
- server.getServerName(), openingVersion) == -1) {
+ if (ZKAssign.transitionNodeOpened(this.server.getZooKeeper(), hri,
+ this.server.getServerName(), this.version) == -1) {
LOG.warn("Completed the OPEN of region " + name +
" but when transitioning from " +
" OPENING to OPENED got a version mismatch, someone else clashed " +
"so now unassigning -- closing region");
- cleanupFailedOpen(region);
- return;
+ } else {
+ result = true;
}
} catch (KeeperException e) {
LOG.error("Failed transitioning node " + name +
" from OPENING to OPENED -- closing region", e);
- cleanupFailedOpen(region);
- return;
- } catch (IOException e) {
- LOG.error("Failed to close " + name +
- " after failing to transition -- closing region", e);
- cleanupFailedOpen(region);
- return;
}
+ return result;
+ }
- // Done! Successful region open
- LOG.debug("Opened " + name);
+ /**
+ * @return Instance of HRegion if successful open else null.
+ */
+ private HRegion openRegion() {
+ HRegion region = null;
+ try {
+ // Instantiate the region. This also periodically tickles our zk OPENING
+ // state so master doesn't timeout this region in transition.
+ region = HRegion.openHRegion(this.regionInfo, this.rsServices.getWAL(),
+ this.server.getConfiguration(), this.rsServices.getFlushRequester(),
+ new Progressable() {
+ public void progress() {
+ // We may lose the znode ownership during the open. Currently its
+ // too hard interrupting ongoing region open. Just let it complete
+ // and check we still have the znode after region open.
+ tickleOpening("open_region_progress");
+ }
+ });
+ } catch (IOException e) {
+ // We failed open. Let our znode expire in regions-in-transition and
+ // Master will assign elsewhere. Presumes nothing to close.
+ LOG.error("Failed open of region=" +
+ this.regionInfo.getRegionNameAsString(), e);
+ }
+ return region;
}
private void cleanupFailedOpen(final HRegion region) throws IOException {
@@ -200,20 +270,58 @@ public class OpenRegionHandler extends E
this.rsServices.removeFromOnlineRegions(regionInfo.getEncodedName());
}
- int transitionZookeeperOfflineToOpening(final String encodedName) {
- // Transition ZK node from OFFLINE to OPENING
+ /**
+ * Transition ZK node from OFFLINE to OPENING.
+ * @param encodedName Name of the znode file (Region encodedName is the znode
+ * name).
+ * @return True if successful transition.
+ */
+ boolean transitionZookeeperOfflineToOpening(final String encodedName) {
// TODO: should also handle transition from CLOSED?
- int openingVersion = -1;
try {
- if ((openingVersion =
ZKAssign.transitionNodeOpening(server.getZooKeeper(),
- regionInfo, server.getServerName())) == -1) {
- LOG.warn("Error transitioning node from OFFLINE to OPENING, " +
- "aborting open");
- }
+ // Initialize the znode version.
+ this.version =
+ ZKAssign.transitionNodeOpening(server.getZooKeeper(),
+ regionInfo, server.getServerName());
} catch (KeeperException e) {
- LOG.error("Error transitioning node from OFFLINE to OPENING for region "
+
+ LOG.error("Error transition from OFFLINE to OPENING for region=" +
encodedName, e);
}
- return openingVersion;
+ boolean b = isGoodVersion();
+ if (!b) {
+ LOG.warn("Failed transition from OFFLINE to OPENING for region=" +
+ encodedName);
+ }
+ return b;
+ }
+
+ /**
+ * Update our OPENING state in zookeeper.
+ * Do this so master doesn't timeout this region-in-transition.
+ * @param context Some context to add to logs if failure
+ * @return True if successful transition.
+ */
+ boolean tickleOpening(final String context) {
+ // If previous checks failed... do not try again.
+ if (!isGoodVersion()) return false;
+ String encodedName = this.regionInfo.getEncodedName();
+ try {
+ this.version =
+ ZKAssign.retransitionNodeOpening(server.getZooKeeper(),
+ this.regionInfo, this.server.getServerName(), this.version);
+ } catch (KeeperException e) {
+ server.abort("Exception refreshing OPENING; region=" + encodedName +
+ ", context=" + context, e);
+ }
+ boolean b = isGoodVersion();
+ if (!b) {
+ LOG.warn("Failed refreshing OPENING; region=" + encodedName +
+ ", context=" + context);
+ }
+ return b;
+ }
+
+ private boolean isGoodVersion() {
+ return this.version != -1;
}
}
