Author: stack
Date: Fri Sep 24 05:46:41 2010
New Revision: 1000715
URL: http://svn.apache.org/viewvc?rev=1000715&view=rev
Log:
HBASE-3028 No basescanner means no GC'ing of split, offlined parent regions
Added:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
Modified:
hbase/trunk/CHANGES.txt
hbase/trunk/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/Threads.java
Modified: hbase/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=1000715&r1=1000714&r2=1000715&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Fri Sep 24 05:46:41 2010
@@ -537,6 +537,7 @@ Release 0.21.0 - Unreleased
HBASE-3026 Fixup of "missing" daughters on split is too aggressive
HBASE-3003 ClassSize constants dont use 'final'
HBASE-3002 Fix zookeepers.sh to work properly with strange JVM options
+ HBASE-3028 No basescanner means no GC'ing of split, offlined parent regions
IMPROVEMENTS
HBASE-1760 Cleanup TODOs in HTable
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java?rev=1000715&r1=1000714&r2=1000715&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
(original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java Fri Sep
24 05:46:41 2010
@@ -529,6 +529,18 @@ public class HRegionInfo extends Version
this.offLine = offLine;
}
+
+ /**
+ * @return True if this is a split parent region.
+ */
+ public boolean isSplitParent() {
+ if (!isSplit()) return false;
+ if (!isOffline()) {
+ LOG.warn("Region is split but NOT offline: " + getRegionNameAsString());
+ }
+ return true;
+ }
+
/**
* @see java.lang.Object#toString()
*/
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java?rev=1000715&r1=1000714&r2=1000715&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
(original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
Fri Sep 24 05:46:41 2010
@@ -182,13 +182,34 @@ public class MetaEditor {
HRegionInfo regionInfo)
throws IOException {
Delete delete = new Delete(regionInfo.getRegionName());
- catalogTracker.waitForMetaServerConnectionDefault().delete(
- CatalogTracker.META_REGION, delete);
-
+ catalogTracker.waitForMetaServerConnectionDefault().
+ delete(CatalogTracker.META_REGION, delete);
LOG.info("Deleted region " + regionInfo.getRegionNameAsString() + " from
META");
}
/**
+ * Deletes daughter reference in offlined split parent.
+ * @param catalogTracker
+ * @param parent Parent row we're to remove daughter reference from
+ * @param qualifier SplitA or SplitB daughter to remove
+ * @param daughter
+ * @throws NotAllMetaRegionsOnlineException
+ * @throws IOException
+ */
+ public static void deleteDaughterReferenceInParent(CatalogTracker
catalogTracker,
+ final HRegionInfo parent, final byte [] qualifier,
+ final HRegionInfo daughter)
+ throws NotAllMetaRegionsOnlineException, IOException {
+ Delete delete = new Delete(parent.getRegionName());
+ delete.deleteColumns(HConstants.CATALOG_FAMILY, qualifier);
+ catalogTracker.waitForMetaServerConnectionDefault().
+ delete(CatalogTracker.META_REGION, delete);
+ LOG.info("Deleted daughter " + daughter.getRegionNameAsString() +
+ " reference " + Bytes.toString(qualifier) + " from " +
+ parent.getRegionNameAsString() + " .META.");
+ }
+
+ /**
* Updates the region information for the specified region in META.
* @param catalogTracker
* @param regionInfo region to be updated in META
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java?rev=1000715&r1=1000714&r2=1000715&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
(original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
Fri Sep 24 05:46:41 2010
@@ -126,10 +126,36 @@ public class MetaReader {
*/
public static Map<HRegionInfo,HServerAddress> fullScan(CatalogTracker
catalogTracker)
throws IOException {
+ final Map<HRegionInfo,HServerAddress> regions =
+ new TreeMap<HRegionInfo,HServerAddress>();
+ Visitor v = new Visitor() {
+ @Override
+ public boolean visit(Result r) throws IOException {
+ if (r == null || r.isEmpty()) return true;
+ Pair<HRegionInfo,HServerAddress> region = metaRowToRegionPair(r);
+ regions.put(region.getFirst(), region.getSecond());
+ return true;
+ }
+ };
+ fullScan(catalogTracker, v);
+ return regions;
+ }
+
+ /**
+ * Performs a full scan of <code>.META.</code>.
+ * <p>
+ * Returns a map of every region to it's currently assigned server, according
+ * to META. If the region does not have an assignment it will have a null
+ * value in the map.
+ * @param catalogTracker
+ * @param visitor
+ * @throws IOException
+ */
+ public static void fullScan(CatalogTracker catalogTracker,
+ final Visitor visitor)
+ throws IOException {
HRegionInterface metaServer =
catalogTracker.waitForMetaServerConnectionDefault();
- Map<HRegionInfo,HServerAddress> allRegions =
- new TreeMap<HRegionInfo,HServerAddress>();
Scan scan = new Scan();
scan.addFamily(HConstants.CATALOG_FAMILY);
long scannerid = metaServer.openScanner(
@@ -137,16 +163,12 @@ public class MetaReader {
try {
Result data;
while((data = metaServer.next(scannerid)) != null) {
- if (!data.isEmpty()) {
- Pair<HRegionInfo,HServerAddress> region =
- metaRowToRegionPair(data);
- allRegions.put(region.getFirst(), region.getSecond());
- }
+ if (!data.isEmpty()) visitor.visit(data);
}
} finally {
metaServer.close(scannerid);
}
- return allRegions;
+ return;
}
/**
@@ -423,4 +445,17 @@ public class MetaReader {
metaServer.close(scannerid);
}
}
-}
+
+ /**
+ * Implementations 'visit' a catalog table row.
+ */
+ public interface Visitor {
+ /**
+ * Visit the catalog table row.
+ * @param r A row from catalog table
+ * @return True if we are to proceed scanning the table, else false if
+ * we are to stop now.
+ */
+ public boolean visit(final Result r) throws IOException;
+ }
+}
\ No newline at end of file
Added:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java?rev=1000715&view=auto
==============================================================================
---
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
(added)
+++
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
Fri Sep 24 05:46:41 2010
@@ -0,0 +1,258 @@
+/**
+ * Copyright 2008 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.Chore;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.catalog.MetaEditor;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.util.Writables;
+
+/**
+ * A janitor for the catalog tables. Scans the <code>.META.</code> catalog
+ * table on a period looking for unused regions to garbage collect.
+ */
+class CatalogJanitor extends Chore {
+ private static final Log LOG =
LogFactory.getLog(CatalogJanitor.class.getName());
+ private final Server server;
+ private final MasterServices services;
+
+ CatalogJanitor(final Server server, final MasterServices services) {
+ super(server.getServerName() + "-CatalogJanitor",
+ server.getConfiguration().getInt("hbase.catalogjanitor.interval",
300000),
+ server);
+ this.server = server;
+ this.services = services;
+ }
+
+ @Override
+ protected boolean initialChore() {
+ try {
+ scan();
+ } catch (IOException e) {
+ LOG.warn("Failed initial scan of catalog table", e);
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ protected void chore() {
+ try {
+ scan();
+ } catch (IOException e) {
+ LOG.warn("Failed scan of catalog table", e);
+ }
+ }
+
+ /**
+ * Run janitorial scan of catalog <code>.META.</code> table looking for
+ * garbage to collect.
+ * @throws IOException
+ */
+ void scan() throws IOException {
+ // TODO: Only works with single .META. region currently. Fix.
+ final AtomicInteger count = new AtomicInteger(0);
+ // Keep Map of found split parents. There are candidates for cleanup.
+ final Map<HRegionInfo, Result> splitParents =
+ new TreeMap<HRegionInfo, Result>();
+ // This visitor collects split parents and counts rows in the .META. table
+ MetaReader.Visitor visitor = new MetaReader.Visitor() {
+ @Override
+ public boolean visit(Result r) throws IOException {
+ if (r == null || r.isEmpty()) return true;
+ count.incrementAndGet();
+ HRegionInfo info = getHRegionInfo(r);
+ if (info.isSplitParent()) splitParents.put(info, r);
+ // Returning true means "keep scanning"
+ return true;
+ }
+ };
+ // Run full scan of .META. catalog table passing in our custom visitor
+ MetaReader.fullScan(this.server.getCatalogTracker(), visitor);
+ // Now work on our list of found parents. See if any we can clean up.
+ int cleaned = 0;
+ for (Map.Entry<HRegionInfo, Result> e : splitParents.entrySet()) {
+ if (cleanParent(e.getKey(), e.getValue())) cleaned++;
+ }
+ LOG.info("Scanned " + count.get() + " catalog row(s) and gc'd " + cleaned +
+ " unreferenced parent region(s)");
+ }
+
+ /**
+ * Get HRegionInfo from passed Map of row values.
+ * @param result Map to do lookup in.
+ * @return Null if not found (and logs fact that expected COL_REGIONINFO
+ * was missing) else deserialized {...@link HRegionInfo}
+ * @throws IOException
+ */
+ static HRegionInfo getHRegionInfo(final Result result)
+ throws IOException {
+ byte [] bytes =
+ result.getValue(HConstants.CATALOG_FAMILY,
HConstants.REGIONINFO_QUALIFIER);
+ if (bytes == null) {
+ LOG.warn("REGIONINFO_QUALIFIER is empty in " + result);
+ return null;
+ }
+ return Writables.getHRegionInfo(bytes);
+ }
+
+ /**
+ * If daughters no longer hold reference to the parents, delete the parent.
+ * @param server HRegionInterface of meta server to talk to
+ * @param parent HRegionInfo of split offlined parent
+ * @param rowContent Content of <code>parent</code> row in
+ * <code>metaRegionName</code>
+ * @return True if we removed <code>parent</code> from meta table and from
+ * the filesystem.
+ * @throws IOException
+ */
+ boolean cleanParent(final HRegionInfo parent,
+ Result rowContent)
+ throws IOException {
+ boolean result = false;
+ // Run checks on each daughter split.
+ boolean hasReferencesA =
+ checkDaughter(parent, rowContent, HConstants.SPLITA_QUALIFIER);
+ boolean hasReferencesB =
+ checkDaughter(parent, rowContent, HConstants.SPLITB_QUALIFIER);
+ if (!hasReferencesA && !hasReferencesB) {
+ LOG.info("Deleting region " + parent.getRegionNameAsString() +
+ " because daughter splits no longer hold references");
+ FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
+ Path rootdir = this.services.getMasterFileSystem().getRootDir();
+ HRegion.deleteRegion(fs, rootdir, parent);
+ MetaEditor.deleteRegion(this.server.getCatalogTracker(), parent);
+ result = true;
+ }
+ return result;
+ }
+
+
+ /**
+ * See if the passed daughter has references in the filesystem to the parent
+ * and if not, remove the note of daughter region in the parent row: its
+ * column info:splitA or info:splitB.
+ * @param parent
+ * @param rowContent
+ * @param qualifier
+ * @return True if this daughter still has references to the parent.
+ * @throws IOException
+ */
+ boolean checkDaughter(final HRegionInfo parent,
+ final Result rowContent, final byte [] qualifier)
+ throws IOException {
+ HRegionInfo hri = getDaughterRegionInfo(rowContent, qualifier);
+ return hasReferences(parent, rowContent, hri, qualifier);
+ }
+
+ /**
+ * Get daughter HRegionInfo out of parent info:splitA/info:splitB columns.
+ * @param result
+ * @param which Whether "info:splitA" or "info:splitB" column
+ * @return Deserialized content of the info:splitA or info:splitB as a
+ * HRegionInfo
+ * @throws IOException
+ */
+ private HRegionInfo getDaughterRegionInfo(final Result result,
+ final byte [] which)
+ throws IOException {
+ byte [] bytes = result.getValue(HConstants.CATALOG_FAMILY, which);
+ return Writables.getHRegionInfoOrNull(bytes);
+ }
+
+ /**
+ * Remove mention of daughter from parent row.
+ * parent row.
+ * @param metaRegionName
+ * @param srvr
+ * @param parent
+ * @param split
+ * @param qualifier
+ * @throws IOException
+ */
+ private void removeDaughterFromParent(final HRegionInfo parent,
+ final HRegionInfo split, final byte [] qualifier)
+ throws IOException {
+ MetaEditor.deleteDaughterReferenceInParent(this.server.getCatalogTracker(),
+ parent, qualifier, split);
+ }
+
+ /**
+ * Checks if a daughter region -- either splitA or splitB -- still holds
+ * references to parent. If not, removes reference to the split from
+ * the parent meta region row so we don't check it any more.
+ * @param parent Parent region name.
+ * @param rowContent Keyed content of the parent row in meta region.
+ * @param split Which column family.
+ * @param qualifier Which of the daughters to look at, splitA or splitB.
+ * @return True if still has references to parent.
+ * @throws IOException
+ */
+ boolean hasReferences(final HRegionInfo parent,
+ final Result rowContent, final HRegionInfo split,
+ final byte [] qualifier)
+ throws IOException {
+ boolean result = false;
+ if (split == null) return result;
+ FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
+ Path rootdir = this.services.getMasterFileSystem().getRootDir();
+ Path tabledir = new Path(rootdir, split.getTableDesc().getNameAsString());
+ for (HColumnDescriptor family: split.getTableDesc().getFamilies()) {
+ Path p = Store.getStoreHomedir(tabledir, split.getEncodedName(),
+ family.getName());
+ // Look for reference files. Call listStatus with anonymous instance of
PathFilter.
+ FileStatus [] ps = fs.listStatus(p,
+ new PathFilter () {
+ public boolean accept(Path path) {
+ return StoreFile.isReference(path);
+ }
+ }
+ );
+
+ if (ps != null && ps.length > 0) {
+ result = true;
+ break;
+ }
+ }
+ if (!result) {
+ removeDaughterFromParent(parent, split, qualifier);
+ }
+ return result;
+ }
+}
\ No newline at end of file
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1000715&r1=1000714&r2=1000715&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
(original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Fri
Sep 24 05:46:41 2010
@@ -159,9 +159,11 @@ implements HMasterInterface, HMasterRegi
ExecutorService executorService;
private LoadBalancer balancer = new LoadBalancer();
- private Chore balancerChore;
+ private Thread balancerChore;
private volatile boolean balance = true;
+ private Thread catalogJanitorChore;
+
/**
* Initializes the HMaster. The steps are as follows:
*
@@ -313,32 +315,29 @@ implements HMasterInterface, HMasterRegi
}
}
+ // Start balancer and meta catalog janitor after meta and regions have
+ // been assigned.
+ this.balancerChore = getAndStartBalancerChore(this);
+ this.catalogJanitorChore =
+ Threads.setDaemonThreadRunning(new CatalogJanitor(this, this));
// Check if we should stop every second.
Sleeper sleeper = new Sleeper(1000, this);
while (!this.stopped) sleeper.sleep();
} catch (Throwable t) {
abort("Unhandled exception. Starting shutdown.", t);
}
+ // Stop balancer and meta catalog janitor
+ if (this.balancerChore != null) this.balancerChore.interrupt();
+ if (this.catalogJanitorChore != null) this.catalogJanitorChore.interrupt();
// Wait for all the remaining region servers to report in IFF we were
// running a cluster shutdown AND we were NOT aborting.
if (!this.abort && this.serverManager.isClusterShutdown()) {
this.serverManager.letRegionServersShutdown();
}
-
- // Clean up and close up shop
- if (this.infoServer != null) {
- LOG.info("Stopping infoServer");
- try {
- this.infoServer.stop();
- } catch (Exception ex) {
- ex.printStackTrace();
- }
- }
- this.rpcServer.stop();
- if (this.balancerChore != null) this.balancerChore.interrupt();
+ stopServiceThreads();
+ // Stop services started up in the constructor.
this.activeMasterManager.stop();
- this.executorService.shutdown();
HConnectionManager.deleteConnection(this.conf, true);
this.zooKeeper.close();
LOG.info("HMaster main thread exiting");
@@ -459,7 +458,6 @@ implements HMasterInterface, HMasterRegi
this.infoServer.setAttribute(MASTER, this);
this.infoServer.start();
}
- this.balancerChore = getAndStartBalancerChore(this);
// Start the server last so everything else is running before we start
// receiving requests.
@@ -480,10 +478,27 @@ implements HMasterInterface, HMasterRegi
}
}
- private static Chore getAndStartBalancerChore(final HMaster master) {
+ private void stopServiceThreads() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Stopping service threads");
+ }
+ this.rpcServer.stop();
+ // Clean up and close up shop
+ if (this.infoServer != null) {
+ LOG.info("Stopping infoServer");
+ try {
+ this.infoServer.stop();
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ }
+ }
+ this.executorService.shutdown();
+ }
+
+ private static Thread getAndStartBalancerChore(final HMaster master) {
String name = master.getServerName() + "-balancerChore";
int period = master.getConfiguration().
- getInt("hbase.master.balancer.period", 3000000);
+ getInt("hbase.balancer.period", 3000000);
// Start up the load balancer chore
Chore chore = new Chore(name, period, master) {
@Override
@@ -491,8 +506,7 @@ implements HMasterInterface, HMasterRegi
master.balance();
}
};
- Threads.setDaemonThreadRunning(chore, name);
- return chore;
+ return Threads.setDaemonThreadRunning(chore);
}
public MapWritable regionServerStartup(final HServerInfo serverInfo)
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java?rev=1000715&r1=1000714&r2=1000715&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
(original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
Fri Sep 24 05:46:41 2010
@@ -128,13 +128,12 @@ public class ServerManager {
this.metrics = new MasterMetrics(master.getServerName());
this.serverMonitorThread = new ServerMonitor(monitorInterval, master);
String n = Thread.currentThread().getName();
- Threads.setDaemonThreadRunning(this.serverMonitorThread,
- n + ".serverMonitor");
- this.logCleaner = new LogCleaner(c.getInt("hbase.master.cleaner.interval",
60 * 1000),
- master, c, this.services.getMasterFileSystem().getFileSystem(),
- this.services.getMasterFileSystem().getOldLogDir());
- Threads.setDaemonThreadRunning(logCleaner,
- n + ".oldLogCleaner");
+ Threads.setDaemonThreadRunning(this.serverMonitorThread, n +
".serverMonitor");
+ this.logCleaner =
+ new LogCleaner(c.getInt("hbase.master.cleaner.interval", 60 * 1000),
+ master, c, this.services.getMasterFileSystem().getFileSystem(),
+ this.services.getMasterFileSystem().getOldLogDir());
+ Threads.setDaemonThreadRunning(logCleaner, n + ".oldLogCleaner");
}
/**
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1000715&r1=1000714&r2=1000715&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
(original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
Fri Sep 24 05:46:41 2010
@@ -2474,22 +2474,6 @@ public class HRegion implements HeapSize
}
/**
- * Delete a region's meta information from the passed
- * <code>meta</code> region. Deletes the row.
- * @param srvr META server to be updated
- * @param metaRegionName Meta region name
- * @param regionName HRegion to remove from <code>meta</code>
- *
- * @throws IOException
- */
- public static void removeRegionFromMETA(final HRegionInterface srvr,
- final byte [] metaRegionName, final byte [] regionName)
- throws IOException {
- Delete delete = new Delete(regionName);
- srvr.delete(metaRegionName, delete);
- }
-
- /**
* Utility method used by HMaster marking regions offlined.
* @param srvr META server to be updated
* @param metaRegionName Meta region name
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java?rev=1000715&r1=1000714&r2=1000715&view=diff
==============================================================================
---
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
(original)
+++
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
Fri Sep 24 05:46:41 2010
@@ -1578,7 +1578,7 @@ public class HLog implements Syncable {
//Either way, the caller should decide what to do. E.g. ignore if this
is the last
//log in sequence.
//TODO is this scenario still possible if the log has been recovered
(i.e. closed)
- LOG.warn("Could not open " + path + " for reading. File is empty" + e);
+ LOG.warn("Could not open " + path + " for reading. File is empty: " +
e);
return;
} else {
throw e;
@@ -1874,4 +1874,4 @@ public class HLog implements Syncable {
}
}
}
-}
\ No newline at end of file
+}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java?rev=1000715&r1=1000714&r2=1000715&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
(original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java Fri Sep
24 05:46:41 2010
@@ -619,7 +619,7 @@ public class FSUtils {
if (!(fs instanceof DistributedFileSystem)) {
return;
}
- LOG.info("Recovering file" + p);
+ LOG.info("Recovering file " + p);
long startWaiting = System.currentTimeMillis();
// Trying recovery
@@ -654,4 +654,4 @@ public class FSUtils {
LOG.info("Finished lease recover attempt for " + p);
}
-}
\ No newline at end of file
+}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/Threads.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/Threads.java?rev=1000715&r1=1000714&r2=1000715&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/Threads.java
(original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/Threads.java Fri Sep
24 05:46:41 2010
@@ -34,6 +34,15 @@ public class Threads {
/**
* Utility method that sets name, daemon status and starts passed thread.
+ * @param t thread to run
+ * @return Returns the passed Thread <code>t</code>.
+ */
+ public static Thread setDaemonThreadRunning(final Thread t) {
+ return setDaemonThreadRunning(t, t.getName());
+ }
+
+ /**
+ * Utility method that sets name, daemon status and starts passed thread.
* @param t thread to frob
* @param name new name
* @return Returns the passed Thread <code>t</code>.
Added:
hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java?rev=1000715&view=auto
==============================================================================
---
hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
(added)
+++
hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
Fri Sep 24 05:46:41 2010
@@ -0,0 +1,228 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.executor.ExecutorService;
+import org.apache.hadoop.hbase.io.Reference;
+import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+public class TestCatalogJanitor {
+
+ /**
+ * Pseudo server for below tests.
+ */
+ class MockServer implements Server {
+ private final Configuration c;
+ private final CatalogTracker ct;
+
+ MockServer(final HBaseTestingUtility htu)
+ throws NotAllMetaRegionsOnlineException, IOException {
+ this.c = htu.getConfiguration();
+ // Set hbase.rootdir into test dir.
+ FileSystem fs = FileSystem.get(this.c);
+ Path rootdir =
+ fs.makeQualified(HBaseTestingUtility.getTestDir(HConstants.HBASE_DIR));
+ this.c.set(HConstants.HBASE_DIR, rootdir.toString());
+ this.ct = Mockito.mock(CatalogTracker.class);
+ HRegionInterface hri = Mockito.mock(HRegionInterface.class);
+ Mockito.when(ct.waitForMetaServerConnectionDefault()).thenReturn(hri);
+ }
+
+ @Override
+ public CatalogTracker getCatalogTracker() {
+ return this.ct;
+ }
+
+ @Override
+ public Configuration getConfiguration() {
+ return this.c;
+ }
+
+ @Override
+ public String getServerName() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public ZooKeeperWatcher getZooKeeper() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public void abort(String why, Throwable e) {
+ // TODO Auto-generated method stub
+ }
+
+ @Override
+ public boolean isStopped() {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public void stop(String why) {
+ // TODO Auto-generated method stub
+ }
+
+ }
+
+ /**
+ * Mock MasterServices for tests below.
+ */
+ class MockMasterServices implements MasterServices {
+ private final MasterFileSystem mfs;
+
+ MockMasterServices(final Server server) throws IOException {
+ this.mfs = new MasterFileSystem(server);
+ }
+
+ @Override
+ public void checkTableModifiable(byte[] tableName) throws IOException {
+ // TODO Auto-generated method stub
+ }
+
+ @Override
+ public AssignmentManager getAssignmentManager() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public ExecutorService getExecutorService() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public MasterFileSystem getMasterFileSystem() {
+ return this.mfs;
+ }
+
+ @Override
+ public ServerManager getServerManager() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ }
+
+ @Test
+ public void testGetHRegionInfo() throws IOException {
+ assertNull(CatalogJanitor.getHRegionInfo(new Result()));
+ List<KeyValue> kvs = new ArrayList<KeyValue>();
+ Result r = new Result(kvs);
+ assertNull(CatalogJanitor.getHRegionInfo(r));
+ byte [] f = HConstants.CATALOG_FAMILY;
+ // Make a key value that doesn't have the expected qualifier.
+ kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY, f,
+ HConstants.SERVER_QUALIFIER, f));
+ r = new Result(kvs);
+ assertNull(CatalogJanitor.getHRegionInfo(r));
+ // Make a key that does not have a regioninfo value.
+ kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY, f,
+ HConstants.REGIONINFO_QUALIFIER, f));
+ boolean exception = false;
+ try {
+ CatalogJanitor.getHRegionInfo(new Result(kvs));
+ } catch (Exception ioe) {
+ exception = true;
+ }
+ assertTrue(exception);
+ // OK, give it what it expects
+ kvs.clear();
+ kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY, f,
+ HConstants.REGIONINFO_QUALIFIER,
+ Writables.getBytes(HRegionInfo.FIRST_META_REGIONINFO)));
+ HRegionInfo hri = CatalogJanitor.getHRegionInfo(new Result(kvs));
+ assertNotNull(hri);
+ assertTrue(hri.equals(HRegionInfo.FIRST_META_REGIONINFO));
+ }
+
+ @Test
+ public void testCleanParent() throws IOException {
+ HBaseTestingUtility htu = new HBaseTestingUtility();
+ Server server = new MockServer(htu);
+ MasterServices services = new MockMasterServices(server);
+ CatalogJanitor janitor = new CatalogJanitor(server, services);
+ // Create regions.
+ HTableDescriptor htd = new HTableDescriptor("table");
+ htd.addFamily(new HColumnDescriptor("family"));
+ HRegionInfo parent =
+ new HRegionInfo(htd, Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
+ HRegionInfo splita =
+ new HRegionInfo(htd, Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
+ HRegionInfo splitb =
+ new HRegionInfo(htd, Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
+ // Test that when both daughter regions are in place, that we do not
+ // remove the parent.
+ List<KeyValue> kvs = new ArrayList<KeyValue>();
+ kvs.add(new KeyValue(parent.getRegionName(), HConstants.CATALOG_FAMILY,
+ HConstants.SPLITA_QUALIFIER, Writables.getBytes(splita)));
+ kvs.add(new KeyValue(parent.getRegionName(), HConstants.CATALOG_FAMILY,
+ HConstants.SPLITB_QUALIFIER, Writables.getBytes(splitb)));
+ Result r = new Result(kvs);
+ // Add a reference under splitA directory so we don't clear out the parent.
+ Path rootdir = services.getMasterFileSystem().getRootDir();
+ Path tabledir =
+ HTableDescriptor.getTableDir(rootdir, htd.getName());
+ Path storedir = Store.getStoreHomedir(tabledir, splita.getEncodedName(),
+ htd.getColumnFamilies()[0].getName());
+ Reference ref = new Reference(Bytes.toBytes("ccc"), Reference.Range.top);
+ long now = System.currentTimeMillis();
+ // Reference name has this format: StoreFile#REF_NAME_PARSER
+ Path p = new Path(storedir, Long.toString(now) + "." +
parent.getEncodedName());
+ FileSystem fs = services.getMasterFileSystem().getFileSystem();
+ ref.write(fs, p);
+ assertFalse(janitor.cleanParent(parent, r));
+ // Remove the reference file and try again.
+ assertTrue(fs.delete(p, true));
+ assertTrue(janitor.cleanParent(parent, r));
+ }
+}
\ No newline at end of file
