Author: apurtell
Date: Thu Jul 16 21:01:31 2009
New Revision: 794855
URL: http://svn.apache.org/viewvc?rev=794855&view=rev
Log:
HBASE-1663
Modified:
hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Migrate.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/util/MigrationTest.java
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt?rev=794855&r1=794854&r2=794855&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt Thu Jul 16 21:01:31 2009
@@ -267,6 +267,9 @@
anymore
HBASE-1658 Remove UI refresh -- its annoying
HBASE-1659 merge tool doesnt take binary regions with \x escape format
+ HBASE-1663 Request compaction only once instead of every time 500ms each
+ time we cycle the hstore.getStorefilesCount() >
+ this.blockingStoreFilesNumber loop
IMPROVEMENTS
HBASE-1089 Add count of regions on filesystem to master UI; add percentage
Modified:
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java?rev=794855&r1=794854&r2=794855&view=diff
==============================================================================
---
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
(original)
+++
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
Thu Jul 16 21:01:31 2009
@@ -229,13 +229,12 @@
finished = true;
for (Store hstore: region.stores.values()) {
if (hstore.getStorefilesCount() > this.blockingStoreFilesNumber) {
- // always request a compaction
- server.compactSplitThread.compactionRequested(region, getName());
// only log once
if (!triggered) {
LOG.info("Too many store files for region " + region + ": " +
hstore.getStorefilesCount() + ", requesting compaction and " +
"waiting");
+ this.server.compactSplitThread.compactionRequested(region,
getName());
triggered = true;
}
// pending compaction, not finished
@@ -247,16 +246,17 @@
}
}
}
- if(triggered && finished) {
+ if (triggered && finished) {
LOG.info("Compaction has completed, we waited " + (count * 500) + "ms,
"
+ "finishing flush of region " + region);
break;
}
}
- if(triggered && !finished) {
+ if (triggered && !finished) {
LOG.warn("Tried to hold up flushing for compactions of region " + region
+
" but have waited longer than " + blockingWaitTime + "ms,
continuing");
}
+
synchronized (regionsInQueue) {
// See comment above for removeFromQueue on why we do not
// take the region out of the set. If removeFromQueue is true, remove it
Modified:
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Migrate.java
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Migrate.java?rev=794855&r1=794854&r2=794855&view=diff
==============================================================================
---
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Migrate.java
(original)
+++
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Migrate.java
Thu Jul 16 21:01:31 2009
@@ -184,9 +184,9 @@
if (version == HBASE_0_1_VERSION ||
Integer.valueOf(versionStr).intValue() < PREVIOUS_VERSION) {
String msg = "Cannot upgrade from " + versionStr + " to " +
- HConstants.FILE_SYSTEM_VERSION + " you must install an earlier hbase,
run " +
- "the upgrade tool, reinstall this version and run this utility again."
+
- MIGRATION_LINK;
+ HConstants.FILE_SYSTEM_VERSION + " you must install an earlier
hbase, run " +
+ "the upgrade tool, reinstall this version and run this utility
again." +
+ MIGRATION_LINK;
System.out.println(msg);
throw new IOException(msg);
}
@@ -217,13 +217,26 @@
Path hbaseRootDir = new Path(conf.get(HConstants.HBASE_DIR));
boolean pre020 = FSUtils.isPre020FileLayout(fs, hbaseRootDir);
if (pre020) {
+ LOG.info("Checking pre020 filesystem is major compacted");
if (!FSUtils.isMajorCompactedPre020(fs, hbaseRootDir)) {
String msg = "All tables must be major compacted before migration." +
MIGRATION_LINK;
System.out.println(msg);
throw new IOException(msg);
}
- // TODO: Rewrite regions.
+ rewrite(fs, hbaseRootDir);
+ }
+ LOG.info("Checking filesystem is major compacted");
+ // Below check is good for both making sure that we are major compacted
+ // but will also fail if not all dirs were rewritten.
+ if (!FSUtils.isMajorCompacted(fs, hbaseRootDir)) {
+ LOG.info("Checking filesystem is major compacted");
+ if (!FSUtils.isMajorCompacted(fs, hbaseRootDir)) {
+ String msg = "All tables must be major compacted before migration." +
+ MIGRATION_LINK;
+ System.out.println(msg);
+ throw new IOException(msg);
+ }
}
// TOOD: Verify all has been brought over from old to new layout.
final MetaUtils utils = new MetaUtils(this.conf);
@@ -269,6 +282,15 @@
utils.shutdown();
}
}
+
+ /*
+ * Rewrite all under hbase root dir.
+ * @param fs
+ * @param hbaseRootDir
+ */
+ private void rewrite(final FileSystem fs, final Path hbaseRootDir) {
+
+ }
/*
* Enable blockcaching on catalog tables.
Modified:
hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/util/MigrationTest.java
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/util/MigrationTest.java?rev=794855&r1=794854&r2=794855&view=diff
==============================================================================
---
hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/util/MigrationTest.java
(original)
+++
hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/util/MigrationTest.java
Thu Jul 16 21:01:31 2009
@@ -61,7 +61,6 @@
FileSystem fs = FileSystem.get(this.conf);
Path hbasedir = loadTestData(fs, rootdir);
assertTrue(fs.exists(hbasedir));
- listPaths(fs, hbasedir, -1);
Migrate migrator = new Migrate(this.conf);
Path qualified = fs.makeQualified(hbasedir);
String uri = qualified.toString();
