Author: stack
Date: Wed Oct 7 05:52:55 2009
New Revision: 822596
URL: http://svn.apache.org/viewvc?rev=822596&view=rev
Log:
HBASE-1887 Update hbase trunk to latests on hadoop 0.21 branch so we can all
test sync/append; some more fixes
Modified:
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/HMaster.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/HBaseClusterTestCase.java
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java
Modified:
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/LocalHBaseCluster.java?rev=822596&r1=822595&r2=822596&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
(original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
Wed Oct 7 05:52:55 2009
@@ -28,13 +28,12 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hbase.client.HBaseAdmin;
-
+import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.util.ReflectionUtils;
/**
* This class creates a single process HBase cluster. One thread is created for
@@ -249,18 +248,14 @@
/**
* Shut down the mini HBase cluster
+ * @throws IOException
*/
- public void shutdown() {
+ public void shutdown() throws IOException {
LOG.debug("Shutting down HBase Cluster");
- // Be careful how the hdfs shutdown thread runs in context where more than
- // one regionserver in the mix.
- Thread shutdownThread = null;
+ // Be careful about how we shutdown hdfs.
synchronized (this.regionThreads) {
for (RegionServerThread t: this.regionThreads) {
- Thread tt = t.getRegionServer().setHDFSShutdownThreadOnExit(null);
- if (shutdownThread == null && tt != null) {
- shutdownThread = tt;
- }
+ t.getRegionServer().setShutdownHDFS(false);
}
}
if(this.master != null) {
@@ -291,7 +286,7 @@
}
}
}
- Threads.shutdown(shutdownThread);
+ FileSystem.closeAll();
LOG.info("Shutdown " +
((this.regionThreads != null)? this.master.getName(): "0 masters") +
" " + this.regionThreads.size() + " region server(s)");
Modified:
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/HMaster.java
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/HMaster.java?rev=822596&r1=822595&r2=822596&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/HMaster.java
(original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/HMaster.java Wed
Oct 7 05:52:55 2009
@@ -595,7 +595,7 @@
Path logsDirPath =
new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);
if (!this.fs.exists(logsDirPath)) {
- if (this.fs.mkdirs(logsDirPath))
+ if (!this.fs.mkdirs(logsDirPath))
throw new IOException("Failed create of " + logsDirPath);
}
FileStatus [] logFolders = this.fs.listStatus(logsDirPath);
Modified:
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java?rev=822596&r1=822595&r2=822596&view=diff
==============================================================================
---
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
(original)
+++
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
Wed Oct 7 05:52:55 2009
@@ -35,7 +35,6 @@
import org.apache.hadoop.hbase.RemoteExceptionHandler;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.regionserver.HRegionServer.ToDoEntry;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Writables;
import org.apache.hadoop.util.StringUtils;
@@ -203,14 +202,14 @@
oldRegionInfo.setSplit(true);
// Inform the HRegionServer that the parent HRegion is no-longer online.
this.server.removeFromOnlineRegions(oldRegionInfo);
-
+
Put put = new Put(oldRegionInfo.getRegionName());
put.add(CATALOG_FAMILY, REGIONINFO_QUALIFIER,
- Writables.getBytes(oldRegionInfo));
+ Writables.getBytes(oldRegionInfo));
put.add(CATALOG_FAMILY, SPLITA_QUALIFIER,
- Writables.getBytes(newRegions[0].getRegionInfo()));
+ Writables.getBytes(newRegions[0].getRegionInfo()));
put.add(CATALOG_FAMILY, SPLITB_QUALIFIER,
- Writables.getBytes(newRegions[1].getRegionInfo()));
+ Writables.getBytes(newRegions[1].getRegionInfo()));
t.put(put);
// Add new regions to META
Modified:
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=822596&r1=822595&r2=822596&view=diff
==============================================================================
---
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
(original)
+++
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
Wed Oct 7 05:52:55 2009
@@ -228,7 +228,7 @@
// The main region server thread.
private Thread regionServerThread;
- // Run HDFS shutdown thread on exit if this is set. We clear this out when
+ // Run HDFS shutdown on exit if this is set. We clear this out when
// doing a restart() to prevent closing of HDFS.
private final AtomicBoolean shutdownHDFS = new AtomicBoolean(true);
@@ -672,11 +672,6 @@
zooKeeperWrapper.close();
- if (shutdownHDFS.get()) {
- runThread(this.hdfsShutdownThread,
- this.conf.getLong("hbase.dfs.shutdown.wait", 30000));
- }
-
LOG.info(Thread.currentThread().getName() + " exiting");
}
@@ -708,31 +703,6 @@
return null;
}
- /**
- * Run and wait on passed thread in HRS context.
- * @param t
- * @param dfsShutdownWait
- */
- public void runThread(final Thread t, final long dfsShutdownWait) {
- if (t == null) {
- return;
- }
- t.start();
- Threads.shutdown(t, dfsShutdownWait);
- }
-
- /**
- * Set the hdfs shutdown thread to run on exit. Pass null to disable
- * running of the shutdown test. Needed by tests.
- * @param t Thread to run. Pass null to disable tests.
- * @return Previous occupant of the shutdown thread position.
- */
- public Thread setHDFSShutdownThreadOnExit(final Thread t) {
- Thread old = this.hdfsShutdownThread;
- this.hdfsShutdownThread = t;
- return old;
- }
-
/*
* Run init. Sets up hlog and starts up all server threads.
* @param c Extra configuration.
@@ -766,9 +736,9 @@
this.fs = FileSystem.get(this.conf);
// Register shutdown hook for HRegionServer, runs an orderly shutdown
- // when a kill signal is recieved
+ // when a kill signal is recieved. Shuts down hdfs too if its supposed.
Runtime.getRuntime().addShutdownHook(new ShutdownThread(this,
- Thread.currentThread()));
+ Thread.currentThread(), this.shutdownHDFS));
this.conf.setBoolean("fs.automatic.close", false);
this.rootDir = new Path(this.conf.get(HConstants.HBASE_DIR));
@@ -786,6 +756,12 @@
}
}
+ public void setShutdownHDFS(final boolean b) {
+ this.shutdownHDFS.set(b);
+ }
+
+ public boolean getShutdownHDFS() {return this.shutdownHDFS.get();}
+
/*
* @param r Region to get RegionLoad for.
* @return RegionLoad instance.
@@ -970,14 +946,18 @@
private static class ShutdownThread extends Thread {
private final HRegionServer instance;
private final Thread mainThread;
+ private final AtomicBoolean shutdownHDFS;
/**
* @param instance
* @param mainThread
+ * @param shutdownHDFS
*/
- public ShutdownThread(HRegionServer instance, Thread mainThread) {
+ public ShutdownThread(final HRegionServer instance, final Thread
mainThread,
+ final AtomicBoolean shutdownHDFS) {
this.instance = instance;
this.mainThread = mainThread;
+ this.shutdownHDFS = shutdownHDFS;
}
@Override
@@ -990,7 +970,7 @@
// Wait for main thread to exit.
Threads.shutdown(this.mainThread);
try {
- FileSystem.closeAll();
+ if (this.shutdownHDFS.get()) FileSystem.closeAll();
} catch (IOException e) {
e.printStackTrace();
}
@@ -999,9 +979,6 @@
}
}
- // We need to call HDFS shutdown when we are done shutting down
- private Thread hdfsShutdownThread;
-
/*
* Inner class that runs on a long period checking if regions need major
* compaction.
Modified:
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/HBaseClusterTestCase.java
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/HBaseClusterTestCase.java?rev=822596&r1=822595&r2=822596&view=diff
==============================================================================
---
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/HBaseClusterTestCase.java
(original)
+++
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/HBaseClusterTestCase.java
Wed Oct 7 05:52:55 2009
@@ -115,32 +115,13 @@
// continue
}
- /*
- * Create dir and set its value into configuration.
- * @param key Create dir under test for this key. Set its fully-qualified
- * value into the conf.
- * @throws IOException
- */
- private void setupDFSConfig(final String key) throws IOException {
- Path basedir =
- new Path(this.conf.get(TEST_DIRECTORY_KEY, "test/build/data"));
- FileSystem fs = FileSystem.get(this.conf);
- Path dir = fs.makeQualified(new Path(basedir, key));
- // Delete if exists. May contain data from old tests.
- if (fs.exists(dir)) if (!fs.delete(dir, true)) throw new
IOException("Delete: " + dir);
- if (!fs.mkdirs(dir)) throw new IOException("Create: " + dir);
- this.conf.set(key, dir.toString());
- }
-
@Override
protected void setUp() throws Exception {
try {
if (this.startDfs) {
- /*
- setupDFSConfig("dfs.namenode.name.dir");
- setupDFSConfig("dfs.datanode.data.dir");
- */
- this.dfsCluster = new MiniDFSCluster(this.conf, 2, true, null);
+ // This spews a bunch of warnings about missing scheme. TODO: fix.
+ this.dfsCluster = new MiniDFSCluster(0, this.conf, 2, true, true, true,
+ null, null, null, null);
// mangle the conf so that the fs parameter points to the minidfs we
// just started up
Modified:
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java?rev=822596&r1=822595&r2=822596&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java
(original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java
Wed Oct 7 05:52:55 2009
@@ -147,7 +147,7 @@
LOG.info("Stopping " + server.toString());
if (!shutdownFS) {
// Stop the running of the hdfs shutdown thread in tests.
- server.getRegionServer().setHDFSShutdownThreadOnExit(null);
+ server.getRegionServer().setShutdownHDFS(false);
}
server.getRegionServer().stop();
return server;
@@ -172,8 +172,9 @@
/**
* Shut down the mini HBase cluster
+ * @throws IOException
*/
- public void shutdown() {
+ public void shutdown() throws IOException {
if (this.hbaseCluster != null) {
this.hbaseCluster.shutdown();
}
