Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=795701&r1=795700&r2=795701&view=diff ============================================================================== --- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original) +++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Mon Jul 20 06:22:20 2009 @@ -1619,7 +1619,16 @@ } finally { this.lock.writeLock().unlock(); } - for(HRegion region: regionsToClose) { + // Close any outstanding scanners. Means they'll get an UnknownScanner + // exception next time they come in. + for (Map.Entry<String, InternalScanner> e: this.scanners.entrySet()) { + try { + e.getValue().close(); + } catch (IOException ioe) { + LOG.warn("Closing scanner " + e.getKey(), ioe); + } + } + for (HRegion region: regionsToClose) { if (LOG.isDebugEnabled()) { LOG.debug("closing region " + Bytes.toString(region.getRegionName())); } @@ -1876,15 +1885,22 @@ } public Result [] next(final long scannerId, int nbRows) throws IOException { - checkOpen(); - List<Result> results = new ArrayList<Result>(); try { String scannerName = String.valueOf(scannerId); InternalScanner s = scanners.get(scannerName); if (s == null) { throw new UnknownScannerException("Name: " + scannerName); } + try { + checkOpen(); + } catch (IOException e) { + // If checkOpen failed, cancel this lease; filesystem is gone or we're + // closing or something. + this.leases.cancelLease(scannerName); + throw e; + } this.leases.renewLease(scannerName); + List<Result> results = new ArrayList<Result>(); for (int i = 0; i < nbRows; i++) { requestCount.incrementAndGet(); // Collect values to be returned here
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/Store.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=795701&r1=795700&r2=795701&view=diff ============================================================================== --- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/Store.java (original) +++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/Store.java Mon Jul 20 06:22:20 2009 @@ -50,7 +50,7 @@ import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.HeapSize; -import org.apache.hadoop.hbase.io.SequenceFile; +import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.hbase.io.hfile.Compression; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileScanner; Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/FSUtils.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/FSUtils.java?rev=795701&r1=795700&r2=795701&view=diff ============================================================================== --- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/FSUtils.java (original) +++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/FSUtils.java Mon Jul 20 06:22:20 2009 @@ -281,22 +281,22 @@ final Path hbaseRootDir) throws IOException { // Presumes any directory under hbase.rootdir is a table. - FileStatus [] directories = fs.listStatus(hbaseRootDir, new DirFilter(fs)); - for (int i = 0; i < directories.length; i++) { + FileStatus [] tableDirs = fs.listStatus(hbaseRootDir, new DirFilter(fs)); + for (int i = 0; i < tableDirs.length; i++) { // Skip the .log directory. All others should be tables. Inside a table, // there are compaction.dir directories to skip. Otherwise, all else // should be regions. Then in each region, should only be family // directories. Under each of these, should be one file only. - Path d = directories[i].getPath(); + Path d = tableDirs[i].getPath(); if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) continue; - FileStatus [] tablesubdirectories = fs.listStatus(d, new DirFilter(fs)); - for (int j = 0; j < tablesubdirectories.length; j++) { - Path dd = tablesubdirectories[j].getPath(); + FileStatus [] regionDirs = fs.listStatus(d, new DirFilter(fs)); + for (int j = 0; j < regionDirs.length; j++) { + Path dd = regionDirs[j].getPath(); if (dd.equals(HConstants.HREGION_COMPACTIONDIR_NAME)) continue; // Else its a region name. Now look in region for families. - FileStatus [] familydirectories = fs.listStatus(dd, new DirFilter(fs)); - for (int k = 0; k < familydirectories.length; k++) { - Path family = familydirectories[k].getPath(); + FileStatus [] familyDirs = fs.listStatus(dd, new DirFilter(fs)); + for (int k = 0; k < familyDirs.length; k++) { + Path family = familyDirs[k].getPath(); // Now in family make sure only one file. FileStatus [] familyStatus = fs.listStatus(family); if (familyStatus.length > 1) { @@ -320,8 +320,8 @@ public static boolean isPre020FileLayout(final FileSystem fs, final Path hbaseRootDir) throws IOException { - Path mapfiles = new Path(new Path(new Path(hbaseRootDir, "-ROOT-"), - "70236052"), "mapfiles"); + Path mapfiles = new Path(new Path(new Path(new Path(hbaseRootDir, "-ROOT-"), + "70236052"), "info"), "mapfiles"); return fs.exists(mapfiles); } @@ -340,22 +340,38 @@ final Path hbaseRootDir) throws IOException { // Presumes any directory under hbase.rootdir is a table. - FileStatus [] directories = fs.listStatus(hbaseRootDir, new DirFilter(fs)); - for (int i = 0; i < directories.length; i++) { + FileStatus [] tableDirs = fs.listStatus(hbaseRootDir, new DirFilter(fs)); + for (int i = 0; i < tableDirs.length; i++) { // Inside a table, there are compaction.dir directories to skip. // Otherwise, all else should be regions. Then in each region, should // only be family directories. Under each of these, should be a mapfile // and info directory and in these only one file. - Path d = directories[i].getPath(); + Path d = tableDirs[i].getPath(); if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) continue; - FileStatus [] tablesubdirectories = fs.listStatus(d, new DirFilter(fs)); - for (int j = 0; j < tablesubdirectories.length; j++) { - Path dd = tablesubdirectories[j].getPath(); + FileStatus [] regionDirs = fs.listStatus(d, new DirFilter(fs)); + for (int j = 0; j < regionDirs.length; j++) { + Path dd = regionDirs[j].getPath(); if (dd.equals(HConstants.HREGION_COMPACTIONDIR_NAME)) continue; // Else its a region name. Now look in region for families. - FileStatus [] familydirectories = fs.listStatus(dd, new DirFilter(fs)); - for (int k = 0; k < familydirectories.length; k++) { - Path family = familydirectories[k].getPath(); + FileStatus [] familyDirs = fs.listStatus(dd, new DirFilter(fs)); + for (int k = 0; k < familyDirs.length; k++) { + Path family = familyDirs[k].getPath(); + FileStatus [] infoAndMapfile = fs.listStatus(family); + // Assert that only info and mapfile in family dir. + if (infoAndMapfile.length != 0 && infoAndMapfile.length != 2) { + LOG.debug(family.toString() + + " has more than just info and mapfile: " + infoAndMapfile.length); + return false; + } + // Make sure directory named info or mapfile. + for (int ll = 0; ll < 2; ll++) { + if (infoAndMapfile[ll].getPath().getName().equals("info") || + infoAndMapfile[ll].getPath().getName().equals("mapfiles")) + continue; + LOG.debug("Unexpected directory name: " + + infoAndMapfile[ll].getPath()); + return false; + } // Now in family, there are 'mapfile' and 'info' subdirs. Just // look in the 'mapfile' subdir. FileStatus [] familyStatus = Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Migrate.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Migrate.java?rev=795701&r1=795700&r2=795701&view=diff ============================================================================== --- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Migrate.java (original) +++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Migrate.java Mon Jul 20 06:22:20 2009 @@ -20,12 +20,14 @@ package org.apache.hadoop.hbase.util; +import java.io.FileNotFoundException; import java.io.IOException; import org.apache.commons.cli.Options; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; @@ -36,6 +38,7 @@ import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.util.FSUtils.DirFilter; import org.apache.hadoop.util.GenericOptionsParser; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; @@ -224,7 +227,7 @@ System.out.println(msg); throw new IOException(msg); } - rewrite(fs, hbaseRootDir); + rewrite(hbaseRootDir); } LOG.info("Checking filesystem is major compacted"); // Below check is good for both making sure that we are major compacted @@ -285,10 +288,57 @@ /* * Rewrite all under hbase root dir. + * Presumes that {...@link FSUtils#isMajorCompactedPre020(FileSystem, Path)} + * has been run before this method is called. * @param fs * @param hbaseRootDir + * @throws IOException */ - private void rewrite(final FileSystem fs, final Path hbaseRootDir) { + private void rewrite(final Path hbaseRootDir) throws IOException { + FileStatus [] tableDirs = fs.listStatus(hbaseRootDir, new DirFilter(fs)); + for (int i = 0; i < tableDirs.length; i++) { + // Inside a table, there are compaction.dir directories to skip. + // Otherwise, all else should be regions. Then in each region, should + // only be family directories. Under each of these, should be a mapfile + // and info directory and in these only one file. + Path d = tableDirs[i].getPath(); + if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) continue; + FileStatus [] regionDirs = fs.listStatus(d, new DirFilter(fs)); + for (int j = 0; j < regionDirs.length; j++) { + Path dd = regionDirs[j].getPath(); + if (dd.equals(HConstants.HREGION_COMPACTIONDIR_NAME)) continue; + // Else its a region name. Now look in region for families. + FileStatus [] familyDirs = fs.listStatus(dd, new DirFilter(fs)); + for (int k = 0; k < familyDirs.length; k++) { + Path family = familyDirs[k].getPath(); + Path mfdir = new Path(family, "mapfiles"); + FileStatus [] mfs = fs.listStatus(mfdir); + if (mfs.length > 1) { + throw new IOException("Should only be one directory in: " + mfdir); + } + Path mf = mfs[0].getPath(); + Path infofile = new Path(new Path(family, "info"), mf.getName()); + rewrite(this.fs, mf, infofile); + } + } + } + } + + /** + * Rewrite the passed mapfile + * @param mapfiledir + * @param infofile + * @throws IOExcepion + */ + public static void rewrite (final FileSystem fs, final Path mapfiledir, + final Path infofile) + throws IOException { + if (!fs.exists(mapfiledir)) { + throw new FileNotFoundException(mapfiledir.toString()); + } + if (!fs.exists(infofile)) { + throw new FileNotFoundException(infofile.toString()); + } } Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/MapFilePerformanceEvaluation.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/MapFilePerformanceEvaluation.java?rev=795701&r1=795700&r2=795701&view=diff ============================================================================== --- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/MapFilePerformanceEvaluation.java (original) +++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/MapFilePerformanceEvaluation.java Mon Jul 20 06:22:20 2009 @@ -30,8 +30,8 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.io.MapFile; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.MapFile; import org.apache.hadoop.io.WritableComparable; /** Added: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/client/TestHTablePool.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/client/TestHTablePool.java?rev=795701&view=auto ============================================================================== --- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/client/TestHTablePool.java (added) +++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/client/TestHTablePool.java Mon Jul 20 06:22:20 2009 @@ -0,0 +1,108 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestCase; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * Tests HTablePool + */ +public class TestHTablePool extends HBaseTestCase { + + public void testTableWithStringName() { + HTablePool pool = new HTablePool((HBaseConfiguration)null, Integer.MAX_VALUE); + String tableName = "testTable"; + + // Request a table from an empty pool + HTable table = pool.getTable(tableName); + assertNotNull(table); + + // Return the table to the pool + pool.putTable(table); + + // Request a table of the same name + HTable sameTable = pool.getTable(tableName); + assertSame(table, sameTable); + } + + public void testTableWithByteArrayName() { + HTablePool pool = new HTablePool((HBaseConfiguration)null, Integer.MAX_VALUE); + byte[] tableName = Bytes.toBytes("testTable"); + + // Request a table from an empty pool + HTable table = pool.getTable(tableName); + assertNotNull(table); + + // Return the table to the pool + pool.putTable(table); + + // Request a table of the same name + HTable sameTable = pool.getTable(tableName); + assertSame(table, sameTable); + } + + public void testTableWithMaxSize() { + HTablePool pool = new HTablePool((HBaseConfiguration)null, 2); + String tableName = "testTable"; + + // Request tables from an empty pool + HTable table1 = pool.getTable(tableName); + HTable table2 = pool.getTable(tableName); + HTable table3 = pool.getTable(tableName); + + // Return the tables to the pool + pool.putTable(table1); + pool.putTable(table2); + // The pool should reject this one since it is already full + pool.putTable(table3); + + // Request tables of the same name + HTable sameTable1 = pool.getTable(tableName); + HTable sameTable2 = pool.getTable(tableName); + HTable sameTable3 = pool.getTable(tableName); + assertSame(table1, sameTable1); + assertSame(table2, sameTable2); + assertNotSame(table3, sameTable3); + } + + public void testTablesWithDifferentNames() { + HTablePool pool = new HTablePool((HBaseConfiguration)null, Integer.MAX_VALUE); + String tableName1 = "testTable1"; + String tableName2 = "testTable2"; + + // Request a table from an empty pool + HTable table1 = pool.getTable(tableName1); + HTable table2 = pool.getTable(tableName2); + assertNotNull(table2); + + // Return the tables to the pool + pool.putTable(table1); + pool.putTable(table2); + + // Request tables of the same names + HTable sameTable1 = pool.getTable(tableName1); + HTable sameTable2 = pool.getTable(tableName2); + assertSame(table1, sameTable1); + assertSame(table2, sameTable2); + } + +} Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java?rev=795701&r1=795700&r2=795701&view=diff ============================================================================== --- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java (original) +++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java Mon Jul 20 06:22:20 2009 @@ -1472,61 +1472,7 @@ } } } - - - /** - * Test for HBASE-810 - * @throws Exception - */ - public void testScanSplitOnRegion() throws Exception { - byte [] tableName = Bytes.toBytes("testtable"); - - HBaseConfiguration hc = initSplit(); - //Setting up region - String method = this.getName(); - initHRegion(tableName, method, hc, new byte [][] {fam3}); - try { - addContent(region, fam3); - region.flushcache(); - final byte [] midkey = region.compactStores(); - assertNotNull(midkey); - Scan scan = new Scan(); - scan.addFamily(fam3); - final InternalScanner s = region.getScanner(scan); - final HRegion regionForThread = region; - - Thread splitThread = new Thread() { - @Override - public void run() { - try { - split(regionForThread, midkey); - } catch (IOException e) { - fail("Unexpected exception " + e); - } - } - }; - splitThread.start(); - for(int i = 0; i < 6; i++) { - try { - Put put = new Put(region.getRegionInfo().getStartKey()); - put.add(fam3, null, Bytes.toBytes("val")); - region.put(put); - Thread.sleep(1000); - } - catch (InterruptedException e) { - fail("Unexpected exception " + e); - } - } - s.next(new ArrayList<KeyValue>()); - s.close(); - } catch(UnknownScannerException ex) { - ex.printStackTrace(); - fail("Got the " + ex); - } - } - - private void assertGet(final HRegion r, final byte [] family, final byte [] k) throws IOException { // Now I have k, get values out and assert they are as expected. Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/util/MigrationTest.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/util/MigrationTest.java?rev=795701&r1=795700&r2=795701&view=diff ============================================================================== --- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/util/MigrationTest.java (original) +++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/util/MigrationTest.java Mon Jul 20 06:22:20 2009 @@ -58,14 +58,14 @@ */ public void testMigration() throws IOException { Path rootdir = getUnitTestdir(getName()); - FileSystem fs = FileSystem.get(this.conf); Path hbasedir = loadTestData(fs, rootdir); assertTrue(fs.exists(hbasedir)); Migrate migrator = new Migrate(this.conf); Path qualified = fs.makeQualified(hbasedir); String uri = qualified.toString(); this.conf.set("hbase.rootdir", uri); - migrator.run(new String [] {"upgrade"}); + int result = migrator.run(new String [] {"upgrade"}); + assertEquals(0, result); } /*
