Author: jimk Date: Fri Aug 10 15:11:05 2007 New Revision: 564780 URL: http://svn.apache.org/viewvc?view=rev&rev=564780 Log: HADOOP-1678 On region split, master should designate which host should serve daughter splits. Phase 1: Master balances load for new regions and when a region server fails.
Added: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HServerLoad.java Modified: lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HBaseAdmin.java lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HConstants.java lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLog.java lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLogEdit.java lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLogKey.java lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HServerInfo.java lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStoreFile.java lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTable.java lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTableDescriptor.java lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestCleanRegionServerExit.java lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestHRegion.java lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestHStoreFile.java lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestRegionServerAbort.java lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestScanner2.java lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestSplit.java lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestToString.java Modified: lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt?view=diff&rev=564780&r1=564779&r2=564780 ============================================================================== --- lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt (original) +++ lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt Fri Aug 10 15:11:05 2007 @@ -86,3 +86,6 @@ 53. HADOOP-1528 HClient for multiple tables - expose close table function 54. HADOOP-1466 Clean up warnings, visibility and javadoc issues in HBase. 55. HADOOP-1662 Make region splits faster + 56. HADOOP-1678 On region split, master should designate which host should + serve daughter splits. Phase 1: Master balances load for new regions and + when a region server fails. Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HBaseAdmin.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HBaseAdmin.java?view=diff&rev=564780&r1=564779&r2=564780 ============================================================================== --- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HBaseAdmin.java (original) +++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HBaseAdmin.java Fri Aug 10 15:11:05 2007 @@ -488,8 +488,9 @@ * @throws IllegalArgumentException - if the table name is reserved */ protected void checkReservedTableName(Text tableName) { - if(tableName.equals(ROOT_TABLE_NAME) - || tableName.equals(META_TABLE_NAME)) { + if(tableName.charAt(0) == '-' || + tableName.charAt(0) == '.' || + tableName.find(",") != -1) { throw new IllegalArgumentException(tableName + " is a reserved table name"); } Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HConstants.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HConstants.java?view=diff&rev=564780&r1=564779&r2=564780 ============================================================================== --- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HConstants.java (original) +++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HConstants.java Fri Aug 10 15:11:05 2007 @@ -92,15 +92,17 @@ // Do we ever need to know all the information that we are storing? /** The root table's name. */ - static final Text ROOT_TABLE_NAME = new Text("--ROOT--"); + static final Text ROOT_TABLE_NAME = new Text("-ROOT-"); /** The META table's name. */ - static final Text META_TABLE_NAME = new Text("--META--"); + static final Text META_TABLE_NAME = new Text(".META."); // Defines for the column names used in both ROOT and META HBase 'meta' tables. - /** The ROOT and META column family */ + /** The ROOT and META column family (string) */ static final String COLUMN_FAMILY_STR = "info:"; + + /** The ROOT and META column family (Text) */ static final Text COLUMN_FAMILY = new Text(COLUMN_FAMILY_STR); /** Array of meta column names */ Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLog.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLog.java?view=diff&rev=564780&r1=564779&r2=564780 ============================================================================== --- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLog.java (original) +++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLog.java Fri Aug 10 15:11:05 2007 @@ -28,6 +28,7 @@ import java.io.*; import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; /** * HLog stores all the edits to the HStore. @@ -73,14 +74,14 @@ SequenceFile.Writer writer; TreeMap<Long, Path> outputfiles = new TreeMap<Long, Path>(); - boolean insideCacheFlush = false; + volatile boolean insideCacheFlush = false; TreeMap<Text, Long> regionToLastFlush = new TreeMap<Text, Long>(); - boolean closed = false; - transient long logSeqNum = 0; + volatile boolean closed = false; + volatile long logSeqNum = 0; long filenum = 0; - transient int numEntries = 0; + AtomicInteger numEntries = new AtomicInteger(0); Integer rollLock = new Integer(0); @@ -125,7 +126,7 @@ logWriters.put(regionName, w); } if (LOG.isDebugEnabled()) { - LOG.debug("Edit " + key.toString()); + LOG.debug("Edit " + key.toString() + "=" + val.toString()); } w.append(key, val); } @@ -173,6 +174,16 @@ fs.mkdirs(dir); rollWriter(); } + + synchronized void setSequenceNumber(long newvalue) { + if (newvalue > logSeqNum) { + if (LOG.isDebugEnabled()) { + LOG.debug("changing sequence number from " + logSeqNum + " to " + + newvalue); + } + logSeqNum = newvalue; + } + } /** * Roll the log writer. That is, start writing log messages to a new file. @@ -266,7 +277,7 @@ } fs.delete(p); } - this.numEntries = 0; + this.numEntries.set(0); } } @@ -343,13 +354,13 @@ new HLogKey(regionName, tableName, row, seqNum[counter++]); HLogEdit logEdit = new HLogEdit(es.getKey(), es.getValue(), timestamp); writer.append(logKey, logEdit); - numEntries++; + numEntries.getAndIncrement(); } } /** @return How many items have been added to the log */ int getNumEntries() { - return numEntries; + return numEntries.get(); } /** @@ -418,7 +429,7 @@ writer.append(new HLogKey(regionName, tableName, HLog.METAROW, logSeqId), new HLogEdit(HLog.METACOLUMN, COMPLETE_CACHEFLUSH.get(), System.currentTimeMillis())); - numEntries++; + numEntries.getAndIncrement(); // Remember the most-recent flush for each region. // This is used to delete obsolete log files. Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLogEdit.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLogEdit.java?view=diff&rev=564780&r1=564779&r2=564780 ============================================================================== --- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLogEdit.java (original) +++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLogEdit.java Fri Aug 10 15:11:05 2007 @@ -72,8 +72,14 @@ /** [EMAIL PROTECTED] */ @Override public String toString() { - return getColumn().toString() + " " + this.getTimestamp() + " " + - new String(getVal()).trim(); + String value = ""; + try { + value = new String(getVal(), HConstants.UTF8_ENCODING); + + } catch (UnsupportedEncodingException e) { + throw new RuntimeException("UTF8 encoding not present?", e); + } + return "(" + getColumn().toString() + "/" + getTimestamp() + "/" + value + ")"; } // Writable Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLogKey.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLogKey.java?view=diff&rev=564780&r1=564779&r2=564780 ============================================================================== --- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLogKey.java (original) +++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLogKey.java Fri Aug 10 15:11:05 2007 @@ -84,7 +84,7 @@ */ @Override public String toString() { - return tablename + "," + regionName + "," + row + "," + logSeqNum; + return tablename + "/" + regionName + "/" + row + "/" + logSeqNum; } /**