Repository: hbase Updated Branches: refs/heads/branch-1 753169a3a -> 8f6388503
http://git-wip-us.apache.org/repos/asf/hbase/blob/8f638850/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java index 760cdc1..1fcb241 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.regionserver.wal; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -88,6 +89,8 @@ public class TestFSHLog { protected static Configuration conf; protected static FileSystem fs; protected static Path dir; + protected static Path rootDir; + protected static Path walRootDir; protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); @Rule @@ -99,8 +102,10 @@ public class TestFSHLog { for (FileStatus dir : entries) { fs.delete(dir.getPath(), true); } - final Path hbaseDir = TEST_UTIL.createRootDir(); - dir = new Path(hbaseDir, currentTest.getMethodName()); + rootDir = TEST_UTIL.createRootDir(); + walRootDir = TEST_UTIL.createWALRootDir(); + dir = new Path(walRootDir, currentTest.getMethodName()); + assertNotEquals(rootDir, walRootDir); } @After @@ -133,6 +138,8 @@ public class TestFSHLog { @AfterClass public static void tearDownAfterClass() throws Exception { + fs.delete(rootDir, true); + fs.delete(walRootDir, true); TEST_UTIL.shutdownMiniCluster(); } @@ -144,7 +151,7 @@ public class TestFSHLog { // test to see whether the coprocessor is loaded or not. FSHLog log = null; try { - log = new FSHLog(fs, FSUtils.getRootDir(conf), dir.toString(), + log = new FSHLog(fs, walRootDir, dir.toString(), HConstants.HREGION_OLDLOGDIR_NAME, conf, null, true, null, null); WALCoprocessorHost host = log.getCoprocessorHost(); Coprocessor c = host.findCoprocessor(SampleRegionWALObserver.class.getName()); @@ -195,7 +202,7 @@ public class TestFSHLog { FSHLog wal1 = null; FSHLog walMeta = null; try { - wal1 = new FSHLog(fs, FSUtils.getRootDir(conf), dir.toString(), + wal1 = new FSHLog(fs, walRootDir, dir.toString(), HConstants.HREGION_OLDLOGDIR_NAME, conf, null, true, null, null); LOG.debug("Log obtained is: " + wal1); Comparator<Path> comp = wal1.LOG_NAME_COMPARATOR; @@ -205,7 +212,7 @@ public class TestFSHLog { assertTrue(comp.compare(p1, p1) == 0); // comparing with different filenum. assertTrue(comp.compare(p1, p2) < 0); - walMeta = new FSHLog(fs, FSUtils.getRootDir(conf), dir.toString(), + walMeta = new FSHLog(fs, walRootDir, dir.toString(), HConstants.HREGION_OLDLOGDIR_NAME, conf, null, true, null, DefaultWALProvider.META_WAL_PROVIDER_ID); Comparator<Path> compMeta = walMeta.LOG_NAME_COMPARATOR; @@ -253,7 +260,7 @@ public class TestFSHLog { LOG.debug("testFindMemStoresEligibleForFlush"); Configuration conf1 = HBaseConfiguration.create(conf); conf1.setInt("hbase.regionserver.maxlogs", 1); - FSHLog wal = new FSHLog(fs, FSUtils.getRootDir(conf1), dir.toString(), + FSHLog wal = new FSHLog(fs, walRootDir, dir.toString(), HConstants.HREGION_OLDLOGDIR_NAME, conf1, null, true, null, null); HTableDescriptor t1 = new HTableDescriptor(TableName.valueOf("t1")).addFamily(new HColumnDescriptor("row")); @@ -330,7 +337,7 @@ public class TestFSHLog { @Test(expected=IOException.class) public void testFailedToCreateWALIfParentRenamed() throws IOException { final String name = "testFailedToCreateWALIfParentRenamed"; - FSHLog log = new FSHLog(fs, FSUtils.getRootDir(conf), name, HConstants.HREGION_OLDLOGDIR_NAME, + FSHLog log = new FSHLog(fs, walRootDir, name, HConstants.HREGION_OLDLOGDIR_NAME, conf, null, true, null, null); long filenum = System.currentTimeMillis(); Path path = log.computeFilename(filenum); @@ -359,13 +366,13 @@ public class TestFSHLog { final byte[] rowName = tableName.getName(); final HTableDescriptor htd = new HTableDescriptor(tableName); htd.addFamily(new HColumnDescriptor("f")); - HRegion r = HRegion.createHRegion(hri, TEST_UTIL.getDefaultRootDirPath(), + HRegion r = HRegion.createHRegion(hri, rootDir, TEST_UTIL.getConfiguration(), htd); HRegion.closeHRegion(r); final int countPerFamily = 10; final MutableBoolean goslow = new MutableBoolean(false); // subclass and doctor a method. - FSHLog wal = new FSHLog(FileSystem.get(conf), TEST_UTIL.getDefaultRootDirPath(), + FSHLog wal = new FSHLog(FileSystem.get(conf), walRootDir, testName, conf) { @Override void atHeadOfRingBufferEventHandlerAppend() { @@ -377,7 +384,7 @@ public class TestFSHLog { } }; HRegion region = HRegion.openHRegion(TEST_UTIL.getConfiguration(), - TEST_UTIL.getTestFileSystem(), TEST_UTIL.getDefaultRootDirPath(), hri, htd, wal); + TEST_UTIL.getTestFileSystem(), rootDir, hri, htd, wal); EnvironmentEdge ee = EnvironmentEdgeManager.getDelegate(); try { List<Put> puts = null; @@ -430,7 +437,7 @@ public class TestFSHLog { SecurityException, IllegalArgumentException, IllegalAccessException { final String name = "testSyncRunnerIndexOverflow"; FSHLog log = - new FSHLog(fs, FSUtils.getRootDir(conf), name, HConstants.HREGION_OLDLOGDIR_NAME, conf, + new FSHLog(fs, walRootDir, name, HConstants.HREGION_OLDLOGDIR_NAME, conf, null, true, null, null); try { Field ringBufferEventHandlerField = FSHLog.class.getDeclaredField("ringBufferEventHandler"); @@ -468,7 +475,7 @@ public class TestFSHLog { final CountDownLatch putFinished = new CountDownLatch(1); try (FSHLog log = - new FSHLog(fs, FSUtils.getRootDir(conf), name, HConstants.HREGION_OLDLOGDIR_NAME, conf, + new FSHLog(fs, walRootDir, name, HConstants.HREGION_OLDLOGDIR_NAME, conf, null, true, null, null)) { log.registerWALActionsListener(new WALActionsListener.Base() { http://git-wip-us.apache.org/repos/asf/hbase/blob/8f638850/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java index 2ccf12b..3f15c2a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java @@ -71,7 +71,8 @@ public class TestLogRollAbort { /* For the split-then-roll test */ private static final Path HBASEDIR = new Path("/hbase"); - private static final Path OLDLOGDIR = new Path(HBASEDIR, HConstants.HREGION_OLDLOGDIR_NAME); + private static final Path HBASELOGDIR = new Path("/hbaselog"); + private static final Path OLDLOGDIR = new Path(HBASELOGDIR, HConstants.HREGION_OLDLOGDIR_NAME); // Need to override this setup so we can edit the config before it gets sent // to the HDFS & HBase cluster startup. @@ -112,6 +113,7 @@ public class TestLogRollAbort { // disable region rebalancing (interferes with log watching) cluster.getMaster().balanceSwitch(false); FSUtils.setRootDir(conf, HBASEDIR); + FSUtils.setWALRootDir(conf, HBASELOGDIR); } @After @@ -183,7 +185,7 @@ public class TestLogRollAbort { public void testLogRollAfterSplitStart() throws IOException { LOG.info("Verify wal roll after split starts will fail."); String logName = "testLogRollAfterSplitStart"; - Path thisTestsDir = new Path(HBASEDIR, DefaultWALProvider.getWALDirectoryName(logName)); + Path thisTestsDir = new Path(HBASELOGDIR, DefaultWALProvider.getWALDirectoryName(logName)); final WALFactory wals = new WALFactory(conf, null, logName); try { @@ -220,7 +222,7 @@ public class TestLogRollAbort { LOG.debug("Renamed region directory: " + rsSplitDir); LOG.debug("Processing the old log files."); - WALSplitter.split(HBASEDIR, rsSplitDir, OLDLOGDIR, fs, conf, wals); + WALSplitter.split(HBASELOGDIR, rsSplitDir, OLDLOGDIR, fs, conf, wals); LOG.debug("Trying to roll the WAL."); try { http://git-wip-us.apache.org/repos/asf/hbase/blob/8f638850/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java index 3eba637..bac1b6f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java @@ -20,14 +20,19 @@ package org.apache.hadoop.hbase.regionserver.wal; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; @@ -53,21 +58,27 @@ public class TestWALActionsListener { new HBaseTestingUtility(); private final static byte[] SOME_BYTES = Bytes.toBytes("t"); - private static FileSystem fs; private static Configuration conf; + private static Path rootDir; + private static Path walRootDir; + private static FileSystem fs; + private static FileSystem walFs; @BeforeClass public static void setUpBeforeClass() throws Exception { conf = TEST_UTIL.getConfiguration(); conf.setInt("hbase.regionserver.maxlogs", 5); - fs = FileSystem.get(conf); - FSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir()); + rootDir = TEST_UTIL.createRootDir(); + walRootDir = TEST_UTIL.createWALRootDir(); + fs = FSUtils.getRootDirFileSystem(conf); + walFs = FSUtils.getWALFileSystem(conf); } @Before public void setUp() throws Exception { - fs.delete(new Path(TEST_UTIL.getDataTestDir(), HConstants.HREGION_LOGDIR_NAME), true); - fs.delete(new Path(TEST_UTIL.getDataTestDir(), HConstants.HREGION_OLDLOGDIR_NAME), true); + fs.delete(rootDir, true); + walFs.delete(new Path(walRootDir, HConstants.HREGION_LOGDIR_NAME), true); + walFs.delete(new Path(walRootDir, HConstants.HREGION_OLDLOGDIR_NAME), true); } @After http://git-wip-us.apache.org/repos/asf/hbase/blob/8f638850/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java index b478b2d..2622f6d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java @@ -70,6 +70,7 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode; @@ -121,6 +122,7 @@ public class TestWALReplay { static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private final EnvironmentEdge ee = EnvironmentEdgeManager.getDelegate(); private Path hbaseRootDir = null; + private Path hbaseWALRootDir = null; private String logName; private Path oldLogDir; private Path logDir; @@ -142,8 +144,12 @@ public class TestWALReplay { TEST_UTIL.startMiniCluster(3); Path hbaseRootDir = TEST_UTIL.getDFSCluster().getFileSystem().makeQualified(new Path("/hbase")); - LOG.info("hbase.rootdir=" + hbaseRootDir); + Path hbaseWALRootDir = + TEST_UTIL.getDFSCluster().getFileSystem().makeQualified(new Path("/hbaselog")); + LOG.info(HConstants.HBASE_DIR + "=" + hbaseRootDir); + LOG.info(HFileSystem.HBASE_WAL_DIR + "=" + hbaseWALRootDir); FSUtils.setRootDir(conf, hbaseRootDir); + FSUtils.setWALRootDir(conf, hbaseWALRootDir); } @AfterClass @@ -156,12 +162,16 @@ public class TestWALReplay { this.conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration()); this.fs = TEST_UTIL.getDFSCluster().getFileSystem(); this.hbaseRootDir = FSUtils.getRootDir(this.conf); - this.oldLogDir = new Path(this.hbaseRootDir, HConstants.HREGION_OLDLOGDIR_NAME); + this.hbaseWALRootDir = FSUtils.getWALRootDir(this.conf); + this.oldLogDir = new Path(this.hbaseWALRootDir, HConstants.HREGION_OLDLOGDIR_NAME); this.logName = DefaultWALProvider.getWALDirectoryName(currentTest.getMethodName() + "-manual"); - this.logDir = new Path(this.hbaseRootDir, logName); + this.logDir = new Path(this.hbaseWALRootDir, logName); if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseRootDir)) { TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true); } + if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseWALRootDir)) { + TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseWALRootDir, true); + } this.mode = (conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false) ? RecoveryMode.LOG_REPLAY : RecoveryMode.LOG_SPLITTING); this.wals = new WALFactory(conf, null, currentTest.getMethodName()); @@ -171,6 +181,7 @@ public class TestWALReplay { public void tearDown() throws Exception { this.wals.close(); TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true); + TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseWALRootDir, true); } /* @@ -296,11 +307,11 @@ public class TestWALReplay { MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(); HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName); - Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName); + Path basedir = FSUtils.getTableDir(hbaseWALRootDir, tableName); deleteDir(basedir); HTableDescriptor htd = createBasic3FamilyHTD(tableName); - HRegion region2 = HRegion.createHRegion(hri, hbaseRootDir, this.conf, htd); + HRegion region2 = HRegion.createHRegion(hri, hbaseWALRootDir, this.conf, htd); HRegion.closeHRegion(region2); final byte [] rowName = tableName.getName(); @@ -326,7 +337,7 @@ public class TestWALReplay { WAL wal3 = createWAL(this.conf); try { - HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal3); + HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseWALRootDir, hri, htd, wal3); long seqid = region.getOpenSeqNum(); // The regions opens with sequenceId as 1. With 6k edits, its sequence number reaches 6k + 1. // When opened, this region would apply 6k edits, and increment the sequenceId by 1 @@ -358,13 +369,13 @@ public class TestWALReplay { final TableName tableName = TableName.valueOf("testRegionMadeOfBulkLoadedFilesOnly"); final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName); - final Path basedir = new Path(this.hbaseRootDir, tableName.getNameAsString()); + final Path basedir = new Path(this.hbaseWALRootDir, tableName.getNameAsString()); deleteDir(basedir); final HTableDescriptor htd = createBasic3FamilyHTD(tableName); - HRegion region2 = HRegion.createHRegion(hri, hbaseRootDir, this.conf, htd); + HRegion region2 = HRegion.createHRegion(hri, hbaseWALRootDir, this.conf, htd); HRegion.closeHRegion(region2); WAL wal = createWAL(this.conf); - HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf); + HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseWALRootDir, hri, htd, wal); byte [] family = htd.getFamilies().iterator().next().getName(); Path f = new Path(basedir, "hfile"); @@ -393,7 +404,7 @@ public class TestWALReplay { WAL wal2 = createWAL(newConf); HRegion region2 = HRegion.openHRegion(newConf, FileSystem.get(newConf), - hbaseRootDir, hri, htd, wal2); + hbaseWALRootDir, hri, htd, wal2); long seqid2 = region2.getOpenSeqNum(); assertTrue(seqid2 > -1); assertEquals(rowsInsertedCount, getScannedCount(region2.getScanner(new Scan()))); @@ -424,14 +435,14 @@ public class TestWALReplay { final TableName tableName = TableName.valueOf("testCompactedBulkLoadedFiles"); final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName); - final Path basedir = new Path(this.hbaseRootDir, tableName.getNameAsString()); + final Path basedir = new Path(this.hbaseWALRootDir, tableName.getNameAsString()); deleteDir(basedir); final HTableDescriptor htd = createBasic3FamilyHTD(tableName); HRegion region2 = HRegion.createHRegion(hri, - hbaseRootDir, this.conf, htd); + hbaseWALRootDir, this.conf, htd); HRegion.closeHRegion(region2); WAL wal = createWAL(this.conf); - HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf); + HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseWALRootDir, hri, htd, wal); // Add an edit so something in the WAL byte [] row = tableName.getName(); @@ -465,7 +476,7 @@ public class TestWALReplay { WAL wal2 = createWAL(newConf); HRegion region2 = HRegion.openHRegion(newConf, FileSystem.get(newConf), - hbaseRootDir, hri, htd, wal2); + hbaseWALRootDir, hri, htd, wal2); long seqid2 = region2.getOpenSeqNum(); assertTrue(seqid2 > -1); assertEquals(rowsInsertedCount, getScannedCount(region2.getScanner(new Scan()))); @@ -495,19 +506,19 @@ public class TestWALReplay { final TableName tableName = TableName.valueOf("testReplayEditsWrittenViaHRegion"); final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName); - final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName); + final Path basedir = FSUtils.getTableDir(this.hbaseWALRootDir, tableName); deleteDir(basedir); final byte[] rowName = tableName.getName(); final int countPerFamily = 10; final HTableDescriptor htd = createBasic3FamilyHTD(tableName); HRegion region3 = HRegion.createHRegion(hri, - hbaseRootDir, this.conf, htd); + hbaseWALRootDir, this.conf, htd); HRegion.closeHRegion(region3); // Write countPerFamily edits into the three families. Do a flush on one // of the families during the load of edits so its seqid is not same as // others to test we do right thing when different seqids. WAL wal = createWAL(this.conf); - HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal); + HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseWALRootDir, hri, htd, wal); long seqid = region.getOpenSeqNum(); boolean first = true; for (HColumnDescriptor hcd: htd.getFamilies()) { @@ -530,7 +541,7 @@ public class TestWALReplay { wal.shutdown(); runWALSplit(this.conf); WAL wal2 = createWAL(this.conf); - HRegion region2 = HRegion.openHRegion(conf, this.fs, hbaseRootDir, hri, htd, wal2); + HRegion region2 = HRegion.openHRegion(conf, this.fs, hbaseWALRootDir, hri, htd, wal2); long seqid2 = region2.getOpenSeqNum(); assertTrue(seqid + result.size() < seqid2); final Result result1b = region2.get(g); @@ -605,19 +616,19 @@ public class TestWALReplay { final TableName tableName = TableName.valueOf("testReplayEditsWrittenViaHRegion"); final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName); - final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName); + final Path basedir = FSUtils.getTableDir(this.hbaseWALRootDir, tableName); deleteDir(basedir); final byte[] rowName = tableName.getName(); final int countPerFamily = 10; final HTableDescriptor htd = createBasic3FamilyHTD(tableName); HRegion region3 = HRegion.createHRegion(hri, - hbaseRootDir, this.conf, htd); + hbaseWALRootDir, this.conf, htd); HRegion.closeHRegion(region3); // Write countPerFamily edits into the three families. Do a flush on one // of the families during the load of edits so its seqid is not same as // others to test we do right thing when different seqids. WAL wal = createWAL(this.conf); - HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal); + HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseWALRootDir, hri, htd, wal); long seqid = region.getOpenSeqNum(); for (HColumnDescriptor hcd: htd.getFamilies()) { addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x"); @@ -650,7 +661,7 @@ public class TestWALReplay { // Let us try to split and recover runWALSplit(this.conf); WAL wal2 = createWAL(this.conf); - HRegion region2 = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal2); + HRegion region2 = HRegion.openHRegion(this.conf, this.fs, hbaseWALRootDir, hri, htd, wal2); long seqid2 = region2.getOpenSeqNum(); assertTrue(seqid + result.size() < seqid2); @@ -690,10 +701,10 @@ public class TestWALReplay { final TableName tableName = TableName.valueOf("testReplayEditsAfterAbortingFlush"); final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName); - final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName); + final Path basedir = FSUtils.getTableDir(this.hbaseWALRootDir, tableName); deleteDir(basedir); final HTableDescriptor htd = createBasic3FamilyHTD(tableName); - HRegion region3 = HRegion.createHRegion(hri, hbaseRootDir, this.conf, htd); + HRegion region3 = HRegion.createHRegion(hri, hbaseWALRootDir, this.conf, htd); region3.close(); region3.getWAL().close(); // Write countPerFamily edits into the three families. Do a flush on one @@ -707,7 +718,7 @@ public class TestWALReplay { customConf.set(DefaultStoreEngine.DEFAULT_STORE_FLUSHER_CLASS_KEY, CustomStoreFlusher.class.getName()); HRegion region = - HRegion.openHRegion(this.hbaseRootDir, hri, htd, wal, customConf, rsServices, null); + HRegion.openHRegion(this.hbaseWALRootDir, hri, htd, wal, customConf, rsServices, null); int writtenRowCount = 10; List<HColumnDescriptor> families = new ArrayList<HColumnDescriptor>( htd.getFamilies()); @@ -761,7 +772,7 @@ public class TestWALReplay { WAL wal2 = createWAL(this.conf); Mockito.doReturn(false).when(rsServices).isAborted(); HRegion region2 = - HRegion.openHRegion(this.hbaseRootDir, hri, htd, wal2, this.conf, rsServices, null); + HRegion.openHRegion(this.hbaseWALRootDir, hri, htd, wal2, this.conf, rsServices, null); scanner = region2.getScanner(new Scan()); assertEquals(writtenRowCount, getScannedCount(scanner)); } @@ -791,12 +802,12 @@ public class TestWALReplay { TableName.valueOf("testReplayEditsWrittenIntoWAL"); final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(); final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName); - final Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName); + final Path basedir = FSUtils.getTableDir(hbaseWALRootDir, tableName); deleteDir(basedir); final HTableDescriptor htd = createBasic3FamilyHTD(tableName); HRegion region2 = HRegion.createHRegion(hri, - hbaseRootDir, this.conf, htd); + hbaseWALRootDir, this.conf, htd); HRegion.closeHRegion(region2); final WAL wal = createWAL(this.conf); final byte[] rowName = tableName.getName(); @@ -890,7 +901,7 @@ public class TestWALReplay { final TableName tableName = TableName.valueOf(currentTest.getMethodName()); final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName); final Path basedir = - FSUtils.getTableDir(this.hbaseRootDir, tableName); + FSUtils.getTableDir(this.hbaseWALRootDir, tableName); deleteDir(basedir); final byte[] rowName = tableName.getName(); final int countPerFamily = 10; @@ -899,7 +910,7 @@ public class TestWALReplay { // Mock the WAL MockWAL wal = createMockWAL(); - HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal); + HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseWALRootDir, hri, htd, wal); for (HColumnDescriptor hcd : htd.getFamilies()) { addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x"); } @@ -920,10 +931,10 @@ public class TestWALReplay { FileStatus[] listStatus = wal.getFiles(); assertNotNull(listStatus); assertTrue(listStatus.length > 0); - WALSplitter.splitLogFile(hbaseRootDir, listStatus[0], + WALSplitter.splitLogFile(hbaseWALRootDir, listStatus[0], this.fs, this.conf, null, null, null, mode, wals); FileStatus[] listStatus1 = this.fs.listStatus( - new Path(FSUtils.getTableDir(hbaseRootDir, tableName), new Path(hri.getEncodedName(), + new Path(FSUtils.getTableDir(hbaseWALRootDir, tableName), new Path(hri.getEncodedName(), "recovered.edits")), new PathFilter() { @Override public boolean accept(Path p) { @@ -951,17 +962,17 @@ public class TestWALReplay { IllegalAccessException { final TableName tableName = TableName.valueOf("testDatalossWhenInputError"); final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName); - final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName); + final Path basedir = FSUtils.getTableDir(this.hbaseWALRootDir, tableName); deleteDir(basedir); final byte[] rowName = tableName.getName(); final int countPerFamily = 10; final HTableDescriptor htd = createBasic1FamilyHTD(tableName); - HRegion region1 = HBaseTestingUtility.createRegionAndWAL(hri, hbaseRootDir, this.conf, htd); + HRegion region1 = HBaseTestingUtility.createRegionAndWAL(hri, hbaseWALRootDir, this.hbaseWALRootDir, this.conf, htd); Path regionDir = region1.getRegionFileSystem().getRegionDir(); HBaseTestingUtility.closeRegionAndWAL(region1); WAL wal = createWAL(this.conf); - HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal); + HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseWALRootDir, hri, htd, wal); for (HColumnDescriptor hcd : htd.getFamilies()) { addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x"); } @@ -1031,12 +1042,12 @@ public class TestWALReplay { HRegion region2; try { // log replay should fail due to the IOException, otherwise we may lose data. - region2 = HRegion.openHRegion(conf, spyFs, hbaseRootDir, hri, htd, wal2); + region2 = HRegion.openHRegion(conf, spyFs, hbaseWALRootDir, hri, htd, wal2); assertEquals(result.size(), region2.get(g).size()); } catch (IOException e) { assertEquals("read over limit", e.getMessage()); } - region2 = HRegion.openHRegion(conf, fs, hbaseRootDir, hri, htd, wal2); + region2 = HRegion.openHRegion(conf, fs, hbaseWALRootDir, hri, htd, wal2); assertEquals(result.size(), region2.get(g).size()); } @@ -1047,11 +1058,11 @@ public class TestWALReplay { final TableName tableName = TableName.valueOf("testReplayEditsWrittenIntoWAL"); final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(); final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName); - final Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName); + final Path basedir = FSUtils.getTableDir(hbaseWALRootDir, tableName); deleteDir(basedir); final HTableDescriptor htd = createBasic1FamilyHTD(tableName); - HRegion region = HBaseTestingUtility.createRegionAndWAL(hri, hbaseRootDir, this.conf, htd); + HRegion region = HBaseTestingUtility.createRegionAndWAL(hri, hbaseWALRootDir, this.hbaseWALRootDir, this.conf, htd); HBaseTestingUtility.closeRegionAndWAL(region); final byte[] family = htd.getColumnFamilies()[0].getName(); final byte[] rowName = tableName.getName(); @@ -1070,12 +1081,12 @@ public class TestWALReplay { first = fs.getFileStatus(smallFile); second = fs.getFileStatus(largeFile); } - WALSplitter.splitLogFile(hbaseRootDir, first, fs, conf, null, null, null, + WALSplitter.splitLogFile(hbaseWALRootDir, first, fs, conf, null, null, null, RecoveryMode.LOG_SPLITTING, wals); - WALSplitter.splitLogFile(hbaseRootDir, second, fs, conf, null, null, null, + WALSplitter.splitLogFile(hbaseWALRootDir, second, fs, conf, null, null, null, RecoveryMode.LOG_SPLITTING, wals); WAL wal = createWAL(this.conf); - region = HRegion.openHRegion(conf, this.fs, hbaseRootDir, hri, htd, wal); + region = HRegion.openHRegion(conf, this.fs, hbaseWALRootDir, hri, htd, wal); assertTrue(region.getOpenSeqNum() > mvcc.getWritePoint()); assertEquals(2, region.get(new Get(rowName)).size()); } @@ -1093,9 +1104,9 @@ public class TestWALReplay { static class MockWAL extends FSHLog { boolean doCompleteCacheFlush = false; - public MockWAL(FileSystem fs, Path rootDir, String logName, Configuration conf) + public MockWAL(FileSystem fs, Path walRootDir, String logName, Configuration conf) throws IOException { - super(fs, rootDir, logName, HConstants.HREGION_OLDLOGDIR_NAME, conf, null, true, null, null); + super(fs, walRootDir, logName, HConstants.HREGION_OLDLOGDIR_NAME, conf, null, true, null, null); } @Override @@ -1115,7 +1126,7 @@ public class TestWALReplay { } private MockWAL createMockWAL() throws IOException { - MockWAL wal = new MockWAL(fs, hbaseRootDir, logName, conf); + MockWAL wal = new MockWAL(fs, hbaseWALRootDir, logName, conf); // Set down maximum recovery so we dfsclient doesn't linger retrying something // long gone. HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1); @@ -1222,7 +1233,7 @@ public class TestWALReplay { */ private Path runWALSplit(final Configuration c) throws IOException { List<Path> splits = WALSplitter.split( - hbaseRootDir, logDir, oldLogDir, FileSystem.get(c), c, wals); + hbaseWALRootDir, logDir, oldLogDir, FSUtils.getWALFileSystem(c), c, wals); // Split should generate only 1 file since there's only 1 region assertEquals("splits=" + splits, 1, splits.size()); // Make sure the file exists @@ -1237,7 +1248,7 @@ public class TestWALReplay { * @throws IOException */ private WAL createWAL(final Configuration c) throws IOException { - FSHLog wal = new FSHLog(FileSystem.get(c), hbaseRootDir, logName, c); + FSHLog wal = new FSHLog(FSUtils.getWALFileSystem(c), hbaseWALRootDir, logName, c); // Set down maximum recovery so we dfsclient doesn't linger retrying something // long gone. HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1); http://git-wip-us.apache.org/repos/asf/hbase/blob/8f638850/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java index 2699292..ef5ad93 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java @@ -39,8 +39,9 @@ import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HDFSBlocksDistribution; -import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.fs.HFileSystem; +import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -381,6 +382,54 @@ public class TestFSUtils { verifyFileInDirWithStoragePolicy("1772"); } + @Test + public void testSetWALRootDir() throws Exception { + HBaseTestingUtility htu = new HBaseTestingUtility(); + Configuration conf = htu.getConfiguration(); + Path p = new Path("file:///hbase/root"); + FSUtils.setWALRootDir(conf, p); + assertEquals(p.toString(), conf.get(HFileSystem.HBASE_WAL_DIR)); + } + + @Test + public void testGetWALRootDir() throws IOException { + HBaseTestingUtility htu = new HBaseTestingUtility(); + Configuration conf = htu.getConfiguration(); + Path root = new Path("file:///hbase/root"); + Path walRoot = new Path("file:///hbase/logroot"); + FSUtils.setRootDir(conf, root); + assertEquals(FSUtils.getRootDir(conf), root); + assertEquals(FSUtils.getWALRootDir(conf), root); + FSUtils.setWALRootDir(conf, walRoot); + assertEquals(FSUtils.getWALRootDir(conf), walRoot); + } + + @Test(expected=IllegalStateException.class) + public void testGetWALRootDirIllegalWALDir() throws IOException { + HBaseTestingUtility htu = new HBaseTestingUtility(); + Configuration conf = htu.getConfiguration(); + Path root = new Path("file:///hbase/root"); + Path invalidWALDir = new Path("file:///hbase/root/logroot"); + FSUtils.setRootDir(conf, root); + FSUtils.setWALRootDir(conf, invalidWALDir); + FSUtils.getWALRootDir(conf); + } + + @Test + public void testRemoveWALRootPath() throws Exception { + HBaseTestingUtility htu = new HBaseTestingUtility(); + Configuration conf = htu.getConfiguration(); + FSUtils.setRootDir(conf, new Path("file:///user/hbase")); + Path testFile = new Path(FSUtils.getRootDir(conf), "test/testfile"); + Path tmpFile = new Path("file:///test/testfile"); + assertEquals(FSUtils.removeWALRootPath(testFile, conf), "test/testfile"); + assertEquals(FSUtils.removeWALRootPath(tmpFile, conf), tmpFile.toString()); + FSUtils.setWALRootDir(conf, new Path("file:///user/hbaseLogDir")); + assertEquals(FSUtils.removeWALRootPath(testFile, conf), testFile.toString()); + Path logFile = new Path(FSUtils.getWALRootDir(conf), "test/testlog"); + assertEquals(FSUtils.removeWALRootPath(logFile, conf), "test/testlog"); + } + private void cleanupFile(FileSystem fileSys, Path name) throws IOException { assertTrue(fileSys.exists(name)); assertTrue(fileSys.delete(name, true)); http://git-wip-us.apache.org/repos/asf/hbase/blob/8f638850/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/IOTestProvider.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/IOTestProvider.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/IOTestProvider.java index d27939d..a044bcf 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/IOTestProvider.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/IOTestProvider.java @@ -102,7 +102,7 @@ public class IOTestProvider implements WALProvider { providerId = DEFAULT_PROVIDER_ID; } final String logPrefix = factory.factoryId + WAL_FILE_NAME_DELIMITER + providerId; - log = new IOTestWAL(FileSystem.get(conf), FSUtils.getRootDir(conf), + log = new IOTestWAL(FSUtils.getWALFileSystem(conf), FSUtils.getWALRootDir(conf), DefaultWALProvider.getWALDirectoryName(factory.factoryId), HConstants.HREGION_OLDLOGDIR_NAME, conf, listeners, true, logPrefix, META_WAL_PROVIDER_ID.equals(providerId) ? META_WAL_PROVIDER_ID : null); http://git-wip-us.apache.org/repos/asf/hbase/blob/8f638850/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java index e928a4d..fa88294 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java @@ -66,6 +66,8 @@ public class TestDefaultWALProvider { protected static Configuration conf; protected static FileSystem fs; + protected static FileSystem walFs; + protected static Path walRootDir; protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); protected MultiVersionConcurrencyControl mvcc; @@ -79,6 +81,7 @@ public class TestDefaultWALProvider { for (FileStatus dir : entries) { fs.delete(dir.getPath(), true); } + walFs.delete(walRootDir, true); } @After @@ -104,13 +107,15 @@ public class TestDefaultWALProvider { TEST_UTIL.startMiniDFSCluster(3); // Set up a working space for our tests. - TEST_UTIL.createRootDir(); + walRootDir = TEST_UTIL.createWALRootDir(); conf = TEST_UTIL.getConfiguration(); - fs = TEST_UTIL.getDFSCluster().getFileSystem(); + fs = FSUtils.getRootDirFileSystem(conf); + walFs = FSUtils.getWALFileSystem(conf); } @AfterClass public static void tearDownAfterClass() throws Exception { + walFs.delete(walRootDir, true); TEST_UTIL.shutdownMiniCluster(); } @@ -121,13 +126,12 @@ public class TestDefaultWALProvider { @Test public void testGetServerNameFromWALDirectoryName() throws IOException { ServerName sn = ServerName.valueOf("hn", 450, 1398); - String hl = FSUtils.getRootDir(conf) + "/" + + String hl = walRootDir + "/" + DefaultWALProvider.getWALDirectoryName(sn.toString()); // Must not throw exception assertNull(DefaultWALProvider.getServerNameFromWALDirectoryName(conf, null)); - assertNull(DefaultWALProvider.getServerNameFromWALDirectoryName(conf, - FSUtils.getRootDir(conf).toUri().toString())); + assertNull(DefaultWALProvider.getServerNameFromWALDirectoryName(conf, walRootDir.toUri().toString())); assertNull(DefaultWALProvider.getServerNameFromWALDirectoryName(conf, "")); assertNull(DefaultWALProvider.getServerNameFromWALDirectoryName(conf, " ")); assertNull(DefaultWALProvider.getServerNameFromWALDirectoryName(conf, hl)); @@ -136,7 +140,7 @@ public class TestDefaultWALProvider { final String wals = "/WALs/"; ServerName parsed = DefaultWALProvider.getServerNameFromWALDirectoryName(conf, - FSUtils.getRootDir(conf).toUri().toString() + wals + sn + + walRootDir.toUri().toString() + wals + sn + "/localhost%2C32984%2C1343316388997.1343316390417"); assertEquals("standard", sn, parsed); @@ -144,7 +148,7 @@ public class TestDefaultWALProvider { assertEquals("subdir", sn, parsed); parsed = DefaultWALProvider.getServerNameFromWALDirectoryName(conf, - FSUtils.getRootDir(conf).toUri().toString() + wals + sn + + walRootDir.toUri().toString() + wals + sn + "-splitting/localhost%3A57020.1340474893931"); assertEquals("split", sn, parsed); } http://git-wip-us.apache.org/repos/asf/hbase/blob/8f638850/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java index 878bb32..9b4a968 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java @@ -84,6 +84,7 @@ public class TestWALFactory { private static MiniDFSCluster cluster; protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); protected static Path hbaseDir; + protected static Path hbaseWALDir; protected FileSystem fs; protected Path dir; @@ -142,6 +143,7 @@ public class TestWALFactory { cluster = TEST_UTIL.getDFSCluster(); hbaseDir = TEST_UTIL.createRootDir(); + hbaseWALDir = TEST_UTIL.createWALRootDir(); } @AfterClass @@ -164,12 +166,12 @@ public class TestWALFactory { final TableName tableName = TableName.valueOf(currentTest.getMethodName()); final byte [] rowName = tableName.getName(); final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1); - final Path logdir = new Path(hbaseDir, + final Path logdir = new Path(hbaseWALDir, DefaultWALProvider.getWALDirectoryName(currentTest.getMethodName())); - Path oldLogDir = new Path(hbaseDir, HConstants.HREGION_OLDLOGDIR_NAME); + Path oldLogDir = new Path(hbaseWALDir, HConstants.HREGION_OLDLOGDIR_NAME); final int howmany = 3; HRegionInfo[] infos = new HRegionInfo[3]; - Path tabledir = FSUtils.getTableDir(hbaseDir, tableName); + Path tabledir = FSUtils.getTableDir(hbaseWALDir, tableName); fs.mkdirs(tabledir); for(int i = 0; i < howmany; i++) { infos[i] = new HRegionInfo(tableName, @@ -203,7 +205,7 @@ public class TestWALFactory { } } wals.shutdown(); - List<Path> splits = WALSplitter.split(hbaseDir, logdir, oldLogDir, fs, conf, wals); + List<Path> splits = WALSplitter.split(hbaseWALDir, logdir, oldLogDir, fs, conf, wals); verifySplits(splits, howmany); } http://git-wip-us.apache.org/repos/asf/hbase/blob/8f638850/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALRootDir.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALRootDir.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALRootDir.java new file mode 100644 index 0000000..e4c0ae8 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALRootDir.java @@ -0,0 +1,148 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.wal; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.fs.HFileSystem; +import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +@Category(MediumTests.class) +public class TestWALRootDir { + private static final Log LOG = LogFactory.getLog(TestWALRootDir.class); + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static Configuration conf; + private static FileSystem fs; + private static FileSystem walFs; + static final TableName tableName = TableName.valueOf("TestWALWALDir"); + private static final byte [] rowName = Bytes.toBytes("row"); + private static final byte [] family = Bytes.toBytes("column"); + private static HTableDescriptor htd; + private static Path walRootDir; + private static Path rootDir; + private static WALFactory wals; + + @Before + public void setUp() throws Exception { + cleanup(); + } + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + conf = TEST_UTIL.getConfiguration(); + TEST_UTIL.startMiniDFSCluster(1); + rootDir = TEST_UTIL.createRootDir(); + walRootDir = TEST_UTIL.createWALRootDir(); + fs = FSUtils.getRootDirFileSystem(conf); + walFs = FSUtils.getWALFileSystem(conf); + htd = new HTableDescriptor(tableName); + htd.addFamily(new HColumnDescriptor(family)); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + cleanup(); + TEST_UTIL.shutdownMiniDFSCluster(); + } + + @Test + public void testWALRootDir() throws Exception { + HRegionInfo regionInfo = new HRegionInfo(tableName); + wals = new WALFactory(conf, null, "testWALRootDir"); + WAL log = wals.getWAL(regionInfo.getEncodedNameAsBytes(), regionInfo.getTable().getNamespace()); + + assertEquals(1, getWALFiles(walFs, walRootDir).size()); + byte [] value = Bytes.toBytes("value"); + WALEdit edit = new WALEdit(); + edit.add(new KeyValue(rowName, family, Bytes.toBytes("1"), + System.currentTimeMillis(), value)); + long txid = log.append(htd,regionInfo, getWalKey(System.currentTimeMillis(), regionInfo, 0), edit, true); + log.sync(txid); + assertEquals("Expect 1 log have been created", 1, getWALFiles(walFs, walRootDir).size()); + log.rollWriter(); + //Create 1 more WAL + assertEquals(2, getWALFiles(walFs, new Path(walRootDir, HConstants.HREGION_LOGDIR_NAME)).size()); + edit.add(new KeyValue(rowName, family, Bytes.toBytes("2"), + System.currentTimeMillis(), value)); + txid = log.append(htd, regionInfo, getWalKey(System.currentTimeMillis(), regionInfo, 1), edit, true); + log.sync(txid); + log.rollWriter(); + log.shutdown(); + + assertEquals("Expect 3 logs in WALs dir", 3, getWALFiles(walFs, new Path(walRootDir, HConstants.HREGION_LOGDIR_NAME)).size()); + } + + protected WALKey getWalKey(final long time, HRegionInfo hri, final long startPoint) { + return new WALKey(hri.getEncodedNameAsBytes(), tableName, time, new MultiVersionConcurrencyControl(startPoint)); + } + + private List<FileStatus> getWALFiles(FileSystem fs, Path dir) + throws IOException { + List<FileStatus> result = new ArrayList<FileStatus>(); + LOG.debug("Scanning " + dir.toString() + " for WAL files"); + + FileStatus[] files = fs.listStatus(dir); + if (files == null) return Collections.emptyList(); + for (FileStatus file : files) { + if (file.isDirectory()) { + // recurse into sub directories + result.addAll(getWALFiles(fs, file.getPath())); + } else { + String name = file.getPath().toString(); + if (!name.startsWith(".")) { + result.add(file); + } + } + } + return result; + } + + private static void cleanup() throws Exception{ + walFs.delete(walRootDir, true); + fs.delete(rootDir, true); + } + +} + http://git-wip-us.apache.org/repos/asf/hbase/blob/8f638850/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java index 4c77d25..3ae155c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java @@ -118,6 +118,7 @@ public class TestWALSplit { protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private Path HBASEDIR; + private Path HBASELOGDIR; private Path WALDIR; private Path OLDLOGDIR; private Path CORRUPTDIR; @@ -180,8 +181,9 @@ public class TestWALSplit { LOG.info("Cleaning up cluster for new test."); fs = TEST_UTIL.getDFSCluster().getFileSystem(); HBASEDIR = TEST_UTIL.createRootDir(); - OLDLOGDIR = new Path(HBASEDIR, HConstants.HREGION_OLDLOGDIR_NAME); - CORRUPTDIR = new Path(HBASEDIR, HConstants.CORRUPT_DIR_NAME); + HBASELOGDIR = TEST_UTIL.createWALRootDir(); + OLDLOGDIR = new Path(HBASELOGDIR, HConstants.HREGION_OLDLOGDIR_NAME); + CORRUPTDIR = new Path(HBASELOGDIR, HConstants.CORRUPT_DIR_NAME); TABLEDIR = FSUtils.getTableDir(HBASEDIR, TABLE_NAME); REGIONS.clear(); Collections.addAll(REGIONS, "bbb", "ccc"); @@ -189,7 +191,7 @@ public class TestWALSplit { this.mode = (conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false) ? RecoveryMode.LOG_REPLAY : RecoveryMode.LOG_SPLITTING); wals = new WALFactory(conf, null, name.getMethodName()); - WALDIR = new Path(HBASEDIR, DefaultWALProvider.getWALDirectoryName(name.getMethodName())); + WALDIR = new Path(HBASELOGDIR, DefaultWALProvider.getWALDirectoryName(name.getMethodName())); //fs.mkdirs(WALDIR); } @@ -205,6 +207,7 @@ public class TestWALSplit { } finally { wals = null; fs.delete(HBASEDIR, true); + fs.delete(HBASELOGDIR, true); } } @@ -1111,7 +1114,7 @@ public class TestWALSplit { useDifferentDFSClient(); WALSplitter.split(HBASEDIR, WALDIR, OLDLOGDIR, fs, conf, wals); - final Path corruptDir = new Path(FSUtils.getRootDir(conf), HConstants.CORRUPT_DIR_NAME); + final Path corruptDir = new Path(FSUtils.getWALRootDir(conf), HConstants.CORRUPT_DIR_NAME); assertEquals(1, fs.listStatus(corruptDir).length); } http://git-wip-us.apache.org/repos/asf/hbase/blob/8f638850/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java index 8af1882..5a3dd87 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java @@ -346,7 +346,7 @@ public final class WALPerformanceEvaluation extends Configured implements Tool { } if (verify) { LOG.info("verifying written log entries."); - Path dir = new Path(FSUtils.getRootDir(getConf()), + Path dir = new Path(FSUtils.getWALRootDir(getConf()), DefaultWALProvider.getWALDirectoryName("wals")); long editCount = 0; FileStatus [] fsss = fs.listStatus(dir);