Repository: hbase Updated Branches: refs/heads/hbase-14439 6d1813a2f -> 815223453
http://git-wip-us.apache.org/repos/asf/hbase/blob/159a67c6/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestExportSnapshotHelpers.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestExportSnapshotHelpers.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestExportSnapshotHelpers.java new file mode 100644 index 0000000..d70e8c0 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestExportSnapshotHelpers.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.fs.legacy.snapshot; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test Export Snapshot Tool helpers + */ +@Category({RegionServerTests.class, SmallTests.class}) +public class TestExportSnapshotHelpers { + private static final Log LOG = LogFactory.getLog(TestExportSnapshotHelpers.class); + + /** + * Verfy the result of getBalanceSplits() method. + * The result are groups of files, used as input list for the "export" mappers. + * All the groups should have similar amount of data. + * + * The input list is a pair of file path and length. + * The getBalanceSplits() function sort it by length, + * and assign to each group a file, going back and forth through the groups. + */ + @Test + public void testBalanceSplit() throws Exception { + // Create a list of files + List<Pair<SnapshotFileInfo, Long>> files = new ArrayList<Pair<SnapshotFileInfo, Long>>(); + for (long i = 0; i <= 20; i++) { + SnapshotFileInfo fileInfo = SnapshotFileInfo.newBuilder() + .setType(SnapshotFileInfo.Type.HFILE) + .setHfile("file-" + i) + .build(); + files.add(new Pair<SnapshotFileInfo, Long>(fileInfo, i)); + } + + // Create 5 groups (total size 210) + // group 0: 20, 11, 10, 1 (total size: 42) + // group 1: 19, 12, 9, 2 (total size: 42) + // group 2: 18, 13, 8, 3 (total size: 42) + // group 3: 17, 12, 7, 4 (total size: 42) + // group 4: 16, 11, 6, 5 (total size: 42) + List<List<Pair<SnapshotFileInfo, Long>>> splits = ExportSnapshot.getBalancedSplits(files, 5); + assertEquals(5, splits.size()); + + String[] split0 = new String[] {"file-20", "file-11", "file-10", "file-1", "file-0"}; + verifyBalanceSplit(splits.get(0), split0, 42); + String[] split1 = new String[] {"file-19", "file-12", "file-9", "file-2"}; + verifyBalanceSplit(splits.get(1), split1, 42); + String[] split2 = new String[] {"file-18", "file-13", "file-8", "file-3"}; + verifyBalanceSplit(splits.get(2), split2, 42); + String[] split3 = new String[] {"file-17", "file-14", "file-7", "file-4"}; + verifyBalanceSplit(splits.get(3), split3, 42); + String[] split4 = new String[] {"file-16", "file-15", "file-6", "file-5"}; + verifyBalanceSplit(splits.get(4), split4, 42); + } + + private void verifyBalanceSplit(final List<Pair<SnapshotFileInfo, Long>> split, + final String[] expected, final long expectedSize) { + assertEquals(expected.length, split.size()); + long totalSize = 0; + for (int i = 0; i < expected.length; ++i) { + Pair<SnapshotFileInfo, Long> fileInfo = split.get(i); + assertEquals(expected[i], fileInfo.getFirst().getHfile()); + totalSize += fileInfo.getSecond(); + } + assertEquals(expectedSize, totalSize); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/159a67c6/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestRestoreSnapshotHelper.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestRestoreSnapshotHelper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestRestoreSnapshotHelper.java new file mode 100644 index 0000000..0154963 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestRestoreSnapshotHelper.java @@ -0,0 +1,181 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.fs.legacy.snapshot.SnapshotManifest; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; +import org.apache.hadoop.hbase.fs.legacy.io.HFileLink; +import org.apache.hadoop.hbase.monitoring.MonitoredTask; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock; +import org.apache.hadoop.hbase.util.FSTableDescriptors; +import org.apache.hadoop.hbase.util.FSUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +/** + * Test the restore/clone operation from a file-system point of view. + */ +@Category({RegionServerTests.class, SmallTests.class}) +public class TestRestoreSnapshotHelper { + private static final Log LOG = LogFactory.getLog(TestRestoreSnapshotHelper.class); + + protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + protected final static String TEST_HFILE = "abc"; + + protected Configuration conf; + protected Path archiveDir; + protected FileSystem fs; + protected Path rootDir; + + protected void setupConf(Configuration conf) { + } + + @Before + public void setup() throws Exception { + rootDir = TEST_UTIL.getDataTestDir("testRestore"); + archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY); + fs = TEST_UTIL.getTestFileSystem(); + conf = TEST_UTIL.getConfiguration(); + setupConf(conf); + FSUtils.setRootDir(conf, rootDir); + } + + @After + public void tearDown() throws Exception { + fs.delete(TEST_UTIL.getDataTestDir(), true); + } + + protected SnapshotMock createSnapshotMock() throws IOException { + return new SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); + } + + @Test + public void testRestore() throws IOException { + restoreAndVerify("snapshot", "testRestore"); + } + + @Test + public void testRestoreWithNamespace() throws IOException { + restoreAndVerify("snapshot", "namespace1:testRestoreWithNamespace"); + } + + private void restoreAndVerify(final String snapshotName, final String tableName) throws IOException { + // Test Rolling-Upgrade like Snapshot. + // half machines writing using v1 and the others using v2 format. + SnapshotMock snapshotMock = createSnapshotMock(); + SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2("snapshot", tableName); + builder.addRegionV1(); + builder.addRegionV2(); + builder.addRegionV2(); + builder.addRegionV1(); + Path snapshotDir = builder.commit(); + HTableDescriptor htd = builder.getTableDescriptor(); + SnapshotDescription desc = builder.getSnapshotDescription(); + + // Test clone a snapshot + HTableDescriptor htdClone = snapshotMock.createHtd("testtb-clone"); + testRestore(snapshotDir, desc, htdClone); + verifyRestore(rootDir, htd, htdClone); + + // Test clone a clone ("link to link") + SnapshotDescription cloneDesc = SnapshotDescription.newBuilder() + .setName("cloneSnapshot") + .setTable("testtb-clone") + .build(); + Path cloneDir = FSUtils.getTableDir(rootDir, htdClone.getTableName()); + HTableDescriptor htdClone2 = snapshotMock.createHtd("testtb-clone2"); + testRestore(cloneDir, cloneDesc, htdClone2); + verifyRestore(rootDir, htd, htdClone2); + } + + private void verifyRestore(final Path rootDir, final HTableDescriptor sourceHtd, + final HTableDescriptor htdClone) throws IOException { + List<String> files = SnapshotTestingUtils.listHFileNames(fs, + FSUtils.getTableDir(rootDir, htdClone.getTableName())); + assertEquals(12, files.size()); + for (int i = 0; i < files.size(); i += 2) { + String linkFile = files.get(i); + String refFile = files.get(i+1); + assertTrue(linkFile + " should be a HFileLink", HFileLink.isHFileLink(linkFile)); + assertTrue(refFile + " should be a Referene", StoreFileInfo.isReference(refFile)); + assertEquals(sourceHtd.getTableName(), HFileLink.getReferencedTableName(linkFile)); + Path refPath = getReferredToFile(refFile); + LOG.debug("get reference name for file " + refFile + " = " + refPath); + assertTrue(refPath.getName() + " should be a HFileLink", HFileLink.isHFileLink(refPath.getName())); + assertEquals(linkFile, refPath.getName()); + } + } + + /** + * Execute the restore operation + * @param snapshotDir The snapshot directory to use as "restore source" + * @param sd The snapshot descriptor + * @param htdClone The HTableDescriptor of the table to restore/clone. + */ + private void testRestore(final Path snapshotDir, final SnapshotDescription sd, + final HTableDescriptor htdClone) throws IOException { + LOG.debug("pre-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir); + FSUtils.logFileSystemState(fs, rootDir, LOG); + + new FSTableDescriptors(conf).createTableDescriptor(htdClone); + RestoreSnapshotHelper helper = getRestoreHelper(rootDir, snapshotDir, sd, htdClone); + helper.restoreStorageRegions(); + + LOG.debug("post-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir); + FSUtils.logFileSystemState(fs, rootDir, LOG); + } + + /** + * Initialize the restore helper, based on the snapshot and table information provided. + */ + private RestoreSnapshotHelper getRestoreHelper(final Path rootDir, final Path snapshotDir, + final SnapshotDescription sd, final HTableDescriptor htdClone) throws IOException { + ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class); + MonitoredTask status = Mockito.mock(MonitoredTask.class); + + SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd); + return new RestoreSnapshotHelper(conf, manifest, + htdClone, monitor, status); + } + + private Path getReferredToFile(final String referenceName) { + Path fakeBasePath = new Path(new Path("table", "region"), "cf"); + return StoreFileInfo.getReferredToFile(new Path(fakeBasePath, referenceName)); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/159a67c6/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestSnapshotFileCache.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestSnapshotFileCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestSnapshotFileCache.java index a92a5bc..64ea9d8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestSnapshotFileCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestSnapshotFileCache.java @@ -41,7 +41,12 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.fs.MasterStorage; +import org.apache.hadoop.hbase.fs.StorageContext; +import org.apache.hadoop.hbase.fs.StorageIdentifier; +import org.apache.hadoop.hbase.fs.legacy.LegacyLayout; import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MasterTests; @@ -66,12 +71,14 @@ public class TestSnapshotFileCache { private static long sequenceId = 0; private static FileSystem fs; private static Path rootDir; + private static MasterStorage<? extends StorageIdentifier> masterStorage; @BeforeClass public static void startCluster() throws Exception { UTIL.startMiniDFSCluster(1); fs = UTIL.getDFSCluster().getFileSystem(); rootDir = UTIL.getDefaultRootDirPath(); + masterStorage = MasterStorage.open(UTIL.getConfiguration(), false); } @AfterClass @@ -82,7 +89,7 @@ public class TestSnapshotFileCache { @After public void cleanupFiles() throws Exception { // cleanup the snapshot directory - Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir); + Path snapshotDir = LegacyLayout.getSnapshotDir(rootDir); fs.delete(snapshotDir, true); } @@ -183,15 +190,16 @@ public class TestSnapshotFileCache { private List<FileStatus> getStoreFilesForSnapshot(SnapshotMock.SnapshotBuilder builder) throws IOException { final List<FileStatus> allStoreFiles = Lists.newArrayList(); - SnapshotReferenceUtil - .visitReferencedFiles(UTIL.getConfiguration(), fs, builder.getSnapshotsDir(), - new SnapshotReferenceUtil.SnapshotVisitor() { - @Override public void storeFile(HRegionInfo regionInfo, String familyName, - SnapshotProtos.SnapshotRegionManifest.StoreFile storeFile) throws IOException { - FileStatus status = mockStoreFile(storeFile.getName()); - allStoreFiles.add(status); - } - }); + masterStorage.visitSnapshotStoreFiles(builder.getSnapshotDescription(), StorageContext.DATA, + new MasterStorage.SnapshotStoreFileVisitor() { + @Override + public void visitSnapshotStoreFile(HBaseProtos.SnapshotDescription snapshot, + StorageContext ctx, HRegionInfo hri, String familyName, + SnapshotProtos.SnapshotRegionManifest.StoreFile storeFile) throws IOException { + FileStatus status = mockStoreFile(storeFile.getName()); + allStoreFiles.add(status); + } + }); return allStoreFiles; } @@ -206,7 +214,7 @@ public class TestSnapshotFileCache { class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector { public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException { Collection<String> files = new HashSet<String>(); - files.addAll(SnapshotReferenceUtil.getHFileNames(UTIL.getConfiguration(), fs, snapshotDir)); + files.addAll(SnapshotReferenceUtil.getHFileNames(masterStorage, snapshotDir.getName())); return files; } }; http://git-wip-us.apache.org/repos/asf/hbase/blob/159a67c6/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestSnapshotManifest.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestSnapshotManifest.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestSnapshotManifest.java new file mode 100644 index 0000000..0112d5e8 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestSnapshotManifest.java @@ -0,0 +1,146 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.fs.legacy.snapshot; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotDataManifest; +import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; +import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException; +import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.ByteStringer; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.io.IOException; + +import static org.junit.Assert.fail; + +@Category({MasterTests.class, SmallTests.class}) +public class TestSnapshotManifest { + private final Log LOG = LogFactory.getLog(getClass()); + + private static final String TABLE_NAME_STR = "testSnapshotManifest"; + private static final TableName TABLE_NAME = TableName.valueOf(TABLE_NAME_STR); + private static final int TEST_NUM_REGIONS = 16000; + + private static HBaseTestingUtility TEST_UTIL; + private Configuration conf; + private FileSystem fs; + private Path rootDir; + private Path snapshotDir; + private SnapshotDescription snapshotDesc; + + @Before + public void setup() throws Exception { + TEST_UTIL = HBaseTestingUtility.createLocalHTU(); + + rootDir = TEST_UTIL.getDataTestDir(TABLE_NAME_STR); + fs = TEST_UTIL.getTestFileSystem(); + conf = TEST_UTIL.getConfiguration(); + + SnapshotTestingUtils.SnapshotMock snapshotMock = + new SnapshotTestingUtils.SnapshotMock(conf, fs, rootDir); + SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = + snapshotMock.createSnapshotV2("snapshot", TABLE_NAME_STR, 0); + snapshotDir = builder.commit(); + snapshotDesc = builder.getSnapshotDescription(); + + SnapshotDataManifest.Builder dataManifestBuilder = + SnapshotDataManifest.newBuilder(); + byte[] startKey = null; + byte[] stopKey = null; + for (int i = 1; i <= TEST_NUM_REGIONS; i++) { + stopKey = Bytes.toBytes(String.format("%016d", i)); + HRegionInfo regionInfo = new HRegionInfo(TABLE_NAME, startKey, stopKey, false); + SnapshotRegionManifest.Builder dataRegionManifestBuilder = + SnapshotRegionManifest.newBuilder(); + + for (HColumnDescriptor hcd: builder.getTableDescriptor().getFamilies()) { + SnapshotRegionManifest.FamilyFiles.Builder family = + SnapshotRegionManifest.FamilyFiles.newBuilder(); + family.setFamilyName(ByteStringer.wrap(hcd.getName())); + for (int j = 0; j < 100; ++j) { + SnapshotRegionManifest.StoreFile.Builder sfManifest = + SnapshotRegionManifest.StoreFile.newBuilder(); + sfManifest.setName(String.format("%032d", i)); + sfManifest.setFileSize((1 + i) * (1 + i) * 1024); + family.addStoreFiles(sfManifest.build()); + } + dataRegionManifestBuilder.addFamilyFiles(family.build()); + } + + dataRegionManifestBuilder.setRegionInfo(HRegionInfo.convert(regionInfo)); + dataManifestBuilder.addRegionManifests(dataRegionManifestBuilder.build()); + + startKey = stopKey; + } + + dataManifestBuilder + .setTableSchema(ProtobufUtil.convertToTableSchema(builder.getTableDescriptor())); + + SnapshotDataManifest dataManifest = dataManifestBuilder.build(); + writeDataManifest(dataManifest); + } + + @After + public void tearDown() throws Exception { + fs.delete(rootDir,true); + } + + @Test + public void testReadSnapshotManifest() throws IOException { + try { + SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc); + fail("fail to test snapshot manifest because message size is too small."); + } catch (CorruptedSnapshotException cse) { + try { + conf.setInt(SnapshotManifest.SNAPSHOT_MANIFEST_SIZE_LIMIT_CONF_KEY, 128 * 1024 * 1024); + SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc); + LOG.info("open snapshot manifest succeed."); + } catch (CorruptedSnapshotException cse2) { + fail("fail to take snapshot because Manifest proto-message too large."); + } + } + } + + private void writeDataManifest(final SnapshotDataManifest manifest) + throws IOException { + FSDataOutputStream stream = fs.create(new Path(snapshotDir, SnapshotManifest.DATA_MANIFEST_NAME)); + try { + manifest.writeTo(stream); + } finally { + stream.close(); + } + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/159a67c6/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java index 2fca12c..c9628f6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java @@ -56,6 +56,9 @@ import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.fs.MasterStorage; import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor; +import org.apache.hadoop.hbase.fs.legacy.snapshot.SnapshotManifest; +import org.apache.hadoop.hbase.fs.legacy.snapshot.SnapshotManifestV1; +import org.apache.hadoop.hbase.fs.legacy.snapshot.SnapshotManifestV2; import org.apache.hadoop.hbase.fs.legacy.io.HFileLink; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.mob.MobUtils; http://git-wip-us.apache.org/repos/asf/hbase/blob/159a67c6/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java deleted file mode 100644 index 6d7d4e1..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java +++ /dev/null @@ -1,376 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.snapshot; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.net.URI; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.CategoryBasedTimeout; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; -import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.FSUtils; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestRule; - -/** - * Test Export Snapshot Tool - */ -@Category({VerySlowMapReduceTests.class, LargeTests.class}) -public class TestExportSnapshot { - @Rule public final TestRule timeout = CategoryBasedTimeout.builder(). - withTimeout(this.getClass()).withLookingForStuckThread(true).build(); - private static final Log LOG = LogFactory.getLog(TestExportSnapshot.class); - - protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - - protected final static byte[] FAMILY = Bytes.toBytes("cf"); - - protected TableName tableName; - private byte[] emptySnapshotName; - private byte[] snapshotName; - private int tableNumFiles; - private Admin admin; - - public static void setUpBaseConf(Configuration conf) { - conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); - conf.setInt("hbase.regionserver.msginterval", 100); - conf.setInt("hbase.client.pause", 250); - conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); - conf.setBoolean("hbase.master.enabletable.roundrobin", true); - conf.setInt("mapreduce.map.maxattempts", 10); - } - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - setUpBaseConf(TEST_UTIL.getConfiguration()); - TEST_UTIL.startMiniCluster(3); - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - TEST_UTIL.shutdownMiniCluster(); - } - - /** - * Create a table and take a snapshot of the table used by the export test. - */ - @Before - public void setUp() throws Exception { - this.admin = TEST_UTIL.getHBaseAdmin(); - - long tid = System.currentTimeMillis(); - tableName = TableName.valueOf("testtb-" + tid); - snapshotName = Bytes.toBytes("snaptb0-" + tid); - emptySnapshotName = Bytes.toBytes("emptySnaptb0-" + tid); - - // create Table - createTable(); - - // Take an empty snapshot - admin.snapshot(emptySnapshotName, tableName); - - // Add some rows - SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY); - tableNumFiles = admin.getTableRegions(tableName).size(); - - // take a snapshot - admin.snapshot(snapshotName, tableName); - } - - protected void createTable() throws Exception { - SnapshotTestingUtils.createPreSplitTable(TEST_UTIL, tableName, 2, FAMILY); - } - - protected interface RegionPredicate { - boolean evaluate(final HRegionInfo regionInfo); - } - - protected RegionPredicate getBypassRegionPredicate() { - return null; - } - - @After - public void tearDown() throws Exception { - TEST_UTIL.deleteTable(tableName); - SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getHBaseAdmin()); - SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL); - } - - /** - * Verify if exported snapshot and copied files matches the original one. - */ - @Test - public void testExportFileSystemState() throws Exception { - testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles); - } - - @Test - public void testExportFileSystemStateWithSkipTmp() throws Exception { - TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, true); - try { - testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles); - } finally { - TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, false); - } - } - - @Test - public void testEmptyExportFileSystemState() throws Exception { - testExportFileSystemState(tableName, emptySnapshotName, emptySnapshotName, 0); - } - - @Test - public void testConsecutiveExports() throws Exception { - Path copyDir = getLocalDestinationDir(); - testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, false); - testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, true); - removeExportDir(copyDir); - } - - @Test - public void testExportWithTargetName() throws Exception { - final byte[] targetName = Bytes.toBytes("testExportWithTargetName"); - testExportFileSystemState(tableName, snapshotName, targetName, tableNumFiles); - } - - private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName, - final byte[] targetName, int filesExpected) throws Exception { - testExportFileSystemState(tableName, snapshotName, targetName, - filesExpected, getHdfsDestinationDir(), false); - } - - protected void testExportFileSystemState(final TableName tableName, - final byte[] snapshotName, final byte[] targetName, int filesExpected, - Path copyDir, boolean overwrite) throws Exception { - testExportFileSystemState(TEST_UTIL.getConfiguration(), tableName, snapshotName, targetName, - filesExpected, TEST_UTIL.getDefaultRootDirPath(), copyDir, - overwrite, getBypassRegionPredicate()); - } - - /** - * Test ExportSnapshot - */ - protected static void testExportFileSystemState(final Configuration conf, final TableName tableName, - final byte[] snapshotName, final byte[] targetName, final int filesExpected, - final Path sourceDir, Path copyDir, final boolean overwrite, - final RegionPredicate bypassregionPredicate) throws Exception { - URI hdfsUri = FileSystem.get(conf).getUri(); - FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration()); - copyDir = copyDir.makeQualified(fs); - - List<String> opts = new ArrayList<String>(); - opts.add("-snapshot"); - opts.add(Bytes.toString(snapshotName)); - opts.add("-copy-to"); - opts.add(copyDir.toString()); - if (targetName != snapshotName) { - opts.add("-target"); - opts.add(Bytes.toString(targetName)); - } - if (overwrite) opts.add("-overwrite"); - - // Export Snapshot - int res = ExportSnapshot.innerMain(conf, opts.toArray(new String[opts.size()])); - assertEquals(0, res); - - // Verify File-System state - FileStatus[] rootFiles = fs.listStatus(copyDir); - assertEquals(filesExpected > 0 ? 2 : 1, rootFiles.length); - for (FileStatus fileStatus: rootFiles) { - String name = fileStatus.getPath().getName(); - assertTrue(fileStatus.isDirectory()); - assertTrue(name.equals(HConstants.SNAPSHOT_DIR_NAME) || - name.equals(HConstants.HFILE_ARCHIVE_DIRECTORY)); - } - - // compare the snapshot metadata and verify the hfiles - final FileSystem hdfs = FileSystem.get(hdfsUri, conf); - final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(snapshotName)); - final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(targetName)); - verifySnapshotDir(hdfs, new Path(sourceDir, snapshotDir), - fs, new Path(copyDir, targetDir)); - Set<String> snapshotFiles = verifySnapshot(conf, fs, copyDir, tableName, - Bytes.toString(targetName), bypassregionPredicate); - assertEquals(filesExpected, snapshotFiles.size()); - } - - /** - * Check that ExportSnapshot will return a failure if something fails. - */ - @Test - public void testExportFailure() throws Exception { - assertEquals(1, runExportAndInjectFailures(snapshotName, false)); - } - - /** - * Check that ExportSnapshot will succede if something fails but the retry succede. - */ - @Test - public void testExportRetry() throws Exception { - assertEquals(0, runExportAndInjectFailures(snapshotName, true)); - } - - /* - * Execute the ExportSnapshot job injecting failures - */ - private int runExportAndInjectFailures(final byte[] snapshotName, boolean retry) - throws Exception { - Path copyDir = getLocalDestinationDir(); - URI hdfsUri = FileSystem.get(TEST_UTIL.getConfiguration()).getUri(); - FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration()); - copyDir = copyDir.makeQualified(fs); - - Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); - conf.setBoolean(ExportSnapshot.CONF_TEST_FAILURE, true); - conf.setBoolean(ExportSnapshot.CONF_TEST_RETRY, retry); - if (!retry) { - conf.setInt("mapreduce.map.maxattempts", 3); - } - // Export Snapshot -// Path sourceDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterStorage().getRootDir(); - Path sourceDir = null; - int res = ExportSnapshot.innerMain(conf, new String[] { - "-snapshot", Bytes.toString(snapshotName), - "-copy-from", sourceDir.toString(), - "-copy-to", copyDir.toString() - }); - return res; - } - - /* - * verify if the snapshot folder on file-system 1 match the one on file-system 2 - */ - protected static void verifySnapshotDir(final FileSystem fs1, final Path root1, - final FileSystem fs2, final Path root2) throws IOException { - assertEquals(listFiles(fs1, root1, root1), listFiles(fs2, root2, root2)); - } - - protected Set<String> verifySnapshot(final FileSystem fs, final Path rootDir, - final TableName tableName, final String snapshotName) throws IOException { - return verifySnapshot(TEST_UTIL.getConfiguration(), fs, rootDir, tableName, - snapshotName, getBypassRegionPredicate()); - } - - /* - * Verify if the files exists - */ - protected static Set<String> verifySnapshot(final Configuration conf, final FileSystem fs, - final Path rootDir, final TableName tableName, final String snapshotName, - final RegionPredicate bypassregionPredicate) throws IOException { - final Path exportedSnapshot = new Path(rootDir, - new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName)); - final Set<String> snapshotFiles = new HashSet<String>(); - final Path exportedArchive = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY); - SnapshotReferenceUtil.visitReferencedFiles(conf, fs, exportedSnapshot, - new SnapshotReferenceUtil.SnapshotVisitor() { - @Override - public void storeFile(final HRegionInfo regionInfo, final String family, - final SnapshotRegionManifest.StoreFile storeFile) throws IOException { - if (bypassregionPredicate != null && bypassregionPredicate.evaluate(regionInfo)) - return; - - String hfile = storeFile.getName(); - snapshotFiles.add(hfile); - if (storeFile.hasReference()) { - // Nothing to do here, we have already the reference embedded - } else { - verifyNonEmptyFile(new Path(exportedArchive, - new Path(FSUtils.getTableDir(new Path("./"), tableName), - new Path(regionInfo.getEncodedName(), new Path(family, hfile))))); - } - } - - private void verifyNonEmptyFile(final Path path) throws IOException { - assertTrue(path + " should exists", fs.exists(path)); - assertTrue(path + " should not be empty", fs.getFileStatus(path).getLen() > 0); - } - }); - - // Verify Snapshot description - SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, exportedSnapshot); - assertTrue(desc.getName().equals(snapshotName)); - assertTrue(desc.getTable().equals(tableName.getNameAsString())); - return snapshotFiles; - } - - private static Set<String> listFiles(final FileSystem fs, final Path root, final Path dir) - throws IOException { - Set<String> files = new HashSet<String>(); - int rootPrefix = root.makeQualified(fs).toString().length(); - FileStatus[] list = FSUtils.listStatus(fs, dir); - if (list != null) { - for (FileStatus fstat: list) { - LOG.debug(fstat.getPath()); - if (fstat.isDirectory()) { - files.addAll(listFiles(fs, root, fstat.getPath())); - } else { - files.add(fstat.getPath().makeQualified(fs).toString().substring(rootPrefix)); - } - } - } - return files; - } - - private Path getHdfsDestinationDir() { -// Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterStorage().getRootDir(); - Path rootDir = null; - Path path = new Path(new Path(rootDir, "export-test"), "export-" + System.currentTimeMillis()); - LOG.info("HDFS export destination path: " + path); - return path; - } - - private Path getLocalDestinationDir() { - Path path = TEST_UTIL.getDataTestDir("local-export-" + System.currentTimeMillis()); - LOG.info("Local export destination path: " + path); - return path; - } - - private static void removeExportDir(final Path path) throws IOException { - FileSystem fs = FileSystem.get(path.toUri(), new Configuration()); - fs.delete(path, true); - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/159a67c6/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotHelpers.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotHelpers.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotHelpers.java deleted file mode 100644 index 2d0088b..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotHelpers.java +++ /dev/null @@ -1,96 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.snapshot; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.apache.hadoop.hbase.testclassification.RegionServerTests; -import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo; -import org.apache.hadoop.hbase.util.Pair; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -/** - * Test Export Snapshot Tool helpers - */ -@Category({RegionServerTests.class, SmallTests.class}) -public class TestExportSnapshotHelpers { - private static final Log LOG = LogFactory.getLog(TestExportSnapshotHelpers.class); - - /** - * Verfy the result of getBalanceSplits() method. - * The result are groups of files, used as input list for the "export" mappers. - * All the groups should have similar amount of data. - * - * The input list is a pair of file path and length. - * The getBalanceSplits() function sort it by length, - * and assign to each group a file, going back and forth through the groups. - */ - @Test - public void testBalanceSplit() throws Exception { - // Create a list of files - List<Pair<SnapshotFileInfo, Long>> files = new ArrayList<Pair<SnapshotFileInfo, Long>>(); - for (long i = 0; i <= 20; i++) { - SnapshotFileInfo fileInfo = SnapshotFileInfo.newBuilder() - .setType(SnapshotFileInfo.Type.HFILE) - .setHfile("file-" + i) - .build(); - files.add(new Pair<SnapshotFileInfo, Long>(fileInfo, i)); - } - - // Create 5 groups (total size 210) - // group 0: 20, 11, 10, 1 (total size: 42) - // group 1: 19, 12, 9, 2 (total size: 42) - // group 2: 18, 13, 8, 3 (total size: 42) - // group 3: 17, 12, 7, 4 (total size: 42) - // group 4: 16, 11, 6, 5 (total size: 42) - List<List<Pair<SnapshotFileInfo, Long>>> splits = ExportSnapshot.getBalancedSplits(files, 5); - assertEquals(5, splits.size()); - - String[] split0 = new String[] {"file-20", "file-11", "file-10", "file-1", "file-0"}; - verifyBalanceSplit(splits.get(0), split0, 42); - String[] split1 = new String[] {"file-19", "file-12", "file-9", "file-2"}; - verifyBalanceSplit(splits.get(1), split1, 42); - String[] split2 = new String[] {"file-18", "file-13", "file-8", "file-3"}; - verifyBalanceSplit(splits.get(2), split2, 42); - String[] split3 = new String[] {"file-17", "file-14", "file-7", "file-4"}; - verifyBalanceSplit(splits.get(3), split3, 42); - String[] split4 = new String[] {"file-16", "file-15", "file-6", "file-5"}; - verifyBalanceSplit(splits.get(4), split4, 42); - } - - private void verifyBalanceSplit(final List<Pair<SnapshotFileInfo, Long>> split, - final String[] expected, final long expectedSize) { - assertEquals(expected.length, split.size()); - long totalSize = 0; - for (int i = 0; i < expected.length; ++i) { - Pair<SnapshotFileInfo, Long> fileInfo = split.get(i); - assertEquals(expected[i], fileInfo.getFirst().getHfile()); - totalSize += fileInfo.getSecond(); - } - assertEquals(expectedSize, totalSize); - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/159a67c6/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java deleted file mode 100644 index 10e820a..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java +++ /dev/null @@ -1,180 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.snapshot; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.testclassification.RegionServerTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; -import org.apache.hadoop.hbase.fs.legacy.io.HFileLink; -import org.apache.hadoop.hbase.monitoring.MonitoredTask; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; -import org.apache.hadoop.hbase.regionserver.StoreFileInfo; -import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock; -import org.apache.hadoop.hbase.util.FSTableDescriptors; -import org.apache.hadoop.hbase.util.FSUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.mockito.Mockito; - -/** - * Test the restore/clone operation from a file-system point of view. - */ -@Category({RegionServerTests.class, SmallTests.class}) -public class TestRestoreSnapshotHelper { - private static final Log LOG = LogFactory.getLog(TestRestoreSnapshotHelper.class); - - protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - protected final static String TEST_HFILE = "abc"; - - protected Configuration conf; - protected Path archiveDir; - protected FileSystem fs; - protected Path rootDir; - - protected void setupConf(Configuration conf) { - } - - @Before - public void setup() throws Exception { - rootDir = TEST_UTIL.getDataTestDir("testRestore"); - archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY); - fs = TEST_UTIL.getTestFileSystem(); - conf = TEST_UTIL.getConfiguration(); - setupConf(conf); - FSUtils.setRootDir(conf, rootDir); - } - - @After - public void tearDown() throws Exception { - fs.delete(TEST_UTIL.getDataTestDir(), true); - } - - protected SnapshotMock createSnapshotMock() throws IOException { - return new SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); - } - - @Test - public void testRestore() throws IOException { - restoreAndVerify("snapshot", "testRestore"); - } - - @Test - public void testRestoreWithNamespace() throws IOException { - restoreAndVerify("snapshot", "namespace1:testRestoreWithNamespace"); - } - - private void restoreAndVerify(final String snapshotName, final String tableName) throws IOException { - // Test Rolling-Upgrade like Snapshot. - // half machines writing using v1 and the others using v2 format. - SnapshotMock snapshotMock = createSnapshotMock(); - SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2("snapshot", tableName); - builder.addRegionV1(); - builder.addRegionV2(); - builder.addRegionV2(); - builder.addRegionV1(); - Path snapshotDir = builder.commit(); - HTableDescriptor htd = builder.getTableDescriptor(); - SnapshotDescription desc = builder.getSnapshotDescription(); - - // Test clone a snapshot - HTableDescriptor htdClone = snapshotMock.createHtd("testtb-clone"); - testRestore(snapshotDir, desc, htdClone); - verifyRestore(rootDir, htd, htdClone); - - // Test clone a clone ("link to link") - SnapshotDescription cloneDesc = SnapshotDescription.newBuilder() - .setName("cloneSnapshot") - .setTable("testtb-clone") - .build(); - Path cloneDir = FSUtils.getTableDir(rootDir, htdClone.getTableName()); - HTableDescriptor htdClone2 = snapshotMock.createHtd("testtb-clone2"); - testRestore(cloneDir, cloneDesc, htdClone2); - verifyRestore(rootDir, htd, htdClone2); - } - - private void verifyRestore(final Path rootDir, final HTableDescriptor sourceHtd, - final HTableDescriptor htdClone) throws IOException { - List<String> files = SnapshotTestingUtils.listHFileNames(fs, - FSUtils.getTableDir(rootDir, htdClone.getTableName())); - assertEquals(12, files.size()); - for (int i = 0; i < files.size(); i += 2) { - String linkFile = files.get(i); - String refFile = files.get(i+1); - assertTrue(linkFile + " should be a HFileLink", HFileLink.isHFileLink(linkFile)); - assertTrue(refFile + " should be a Referene", StoreFileInfo.isReference(refFile)); - assertEquals(sourceHtd.getTableName(), HFileLink.getReferencedTableName(linkFile)); - Path refPath = getReferredToFile(refFile); - LOG.debug("get reference name for file " + refFile + " = " + refPath); - assertTrue(refPath.getName() + " should be a HFileLink", HFileLink.isHFileLink(refPath.getName())); - assertEquals(linkFile, refPath.getName()); - } - } - - /** - * Execute the restore operation - * @param snapshotDir The snapshot directory to use as "restore source" - * @param sd The snapshot descriptor - * @param htdClone The HTableDescriptor of the table to restore/clone. - */ - private void testRestore(final Path snapshotDir, final SnapshotDescription sd, - final HTableDescriptor htdClone) throws IOException { - LOG.debug("pre-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir); - FSUtils.logFileSystemState(fs, rootDir, LOG); - - new FSTableDescriptors(conf).createTableDescriptor(htdClone); - RestoreSnapshotHelper helper = getRestoreHelper(rootDir, snapshotDir, sd, htdClone); - helper.restoreStorageRegions(); - - LOG.debug("post-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir); - FSUtils.logFileSystemState(fs, rootDir, LOG); - } - - /** - * Initialize the restore helper, based on the snapshot and table information provided. - */ - private RestoreSnapshotHelper getRestoreHelper(final Path rootDir, final Path snapshotDir, - final SnapshotDescription sd, final HTableDescriptor htdClone) throws IOException { - ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class); - MonitoredTask status = Mockito.mock(MonitoredTask.class); - - SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd); - return new RestoreSnapshotHelper(conf, manifest, - htdClone, monitor, status); - } - - private Path getReferredToFile(final String referenceName) { - Path fakeBasePath = new Path(new Path("table", "region"), "cf"); - return StoreFileInfo.getReferredToFile(new Path(fakeBasePath, referenceName)); - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/159a67c6/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotManifest.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotManifest.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotManifest.java deleted file mode 100644 index 835f92e..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotManifest.java +++ /dev/null @@ -1,145 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.snapshot; - -import com.google.protobuf.InvalidProtocolBufferException; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; -import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotDataManifest; -import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; -import org.apache.hadoop.hbase.testclassification.MasterTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.apache.hadoop.hbase.util.ByteStringer; -import org.apache.hadoop.hbase.util.Bytes; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -import java.io.IOException; - -import static org.junit.Assert.fail; - -@Category({MasterTests.class, SmallTests.class}) -public class TestSnapshotManifest { - private final Log LOG = LogFactory.getLog(getClass()); - - private static final String TABLE_NAME_STR = "testSnapshotManifest"; - private static final TableName TABLE_NAME = TableName.valueOf(TABLE_NAME_STR); - private static final int TEST_NUM_REGIONS = 16000; - - private static HBaseTestingUtility TEST_UTIL; - private Configuration conf; - private FileSystem fs; - private Path rootDir; - private Path snapshotDir; - private SnapshotDescription snapshotDesc; - - @Before - public void setup() throws Exception { - TEST_UTIL = HBaseTestingUtility.createLocalHTU(); - - rootDir = TEST_UTIL.getDataTestDir(TABLE_NAME_STR); - fs = TEST_UTIL.getTestFileSystem(); - conf = TEST_UTIL.getConfiguration(); - - SnapshotTestingUtils.SnapshotMock snapshotMock = - new SnapshotTestingUtils.SnapshotMock(conf, fs, rootDir); - SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = - snapshotMock.createSnapshotV2("snapshot", TABLE_NAME_STR, 0); - snapshotDir = builder.commit(); - snapshotDesc = builder.getSnapshotDescription(); - - SnapshotDataManifest.Builder dataManifestBuilder = - SnapshotDataManifest.newBuilder(); - byte[] startKey = null; - byte[] stopKey = null; - for (int i = 1; i <= TEST_NUM_REGIONS; i++) { - stopKey = Bytes.toBytes(String.format("%016d", i)); - HRegionInfo regionInfo = new HRegionInfo(TABLE_NAME, startKey, stopKey, false); - SnapshotRegionManifest.Builder dataRegionManifestBuilder = - SnapshotRegionManifest.newBuilder(); - - for (HColumnDescriptor hcd: builder.getTableDescriptor().getFamilies()) { - SnapshotRegionManifest.FamilyFiles.Builder family = - SnapshotRegionManifest.FamilyFiles.newBuilder(); - family.setFamilyName(ByteStringer.wrap(hcd.getName())); - for (int j = 0; j < 100; ++j) { - SnapshotRegionManifest.StoreFile.Builder sfManifest = - SnapshotRegionManifest.StoreFile.newBuilder(); - sfManifest.setName(String.format("%032d", i)); - sfManifest.setFileSize((1 + i) * (1 + i) * 1024); - family.addStoreFiles(sfManifest.build()); - } - dataRegionManifestBuilder.addFamilyFiles(family.build()); - } - - dataRegionManifestBuilder.setRegionInfo(HRegionInfo.convert(regionInfo)); - dataManifestBuilder.addRegionManifests(dataRegionManifestBuilder.build()); - - startKey = stopKey; - } - - dataManifestBuilder - .setTableSchema(ProtobufUtil.convertToTableSchema(builder.getTableDescriptor())); - - SnapshotDataManifest dataManifest = dataManifestBuilder.build(); - writeDataManifest(dataManifest); - } - - @After - public void tearDown() throws Exception { - fs.delete(rootDir,true); - } - - @Test - public void testReadSnapshotManifest() throws IOException { - try { - SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc); - fail("fail to test snapshot manifest because message size is too small."); - } catch (CorruptedSnapshotException cse) { - try { - conf.setInt(SnapshotManifest.SNAPSHOT_MANIFEST_SIZE_LIMIT_CONF_KEY, 128 * 1024 * 1024); - SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc); - LOG.info("open snapshot manifest succeed."); - } catch (CorruptedSnapshotException cse2) { - fail("fail to take snapshot because Manifest proto-message too large."); - } - } - } - - private void writeDataManifest(final SnapshotDataManifest manifest) - throws IOException { - FSDataOutputStream stream = fs.create(new Path(snapshotDir, SnapshotManifest.DATA_MANIFEST_NAME)); - try { - manifest.writeTo(stream); - } finally { - stream.close(); - } - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/159a67c6/src/main/asciidoc/_chapters/ops_mgt.adoc ---------------------------------------------------------------------- diff --git a/src/main/asciidoc/_chapters/ops_mgt.adoc b/src/main/asciidoc/_chapters/ops_mgt.adoc index c1db106..dde1779 100644 --- a/src/main/asciidoc/_chapters/ops_mgt.adoc +++ b/src/main/asciidoc/_chapters/ops_mgt.adoc @@ -2063,7 +2063,7 @@ To copy a snapshot called MySnapshot to an HBase cluster srv2 (hdfs:///srv2:8082 [source,bourne] ---- -$ bin/hbase org.apache.hadoop.hbase.snapshot.ExportSnapshot -snapshot MySnapshot -copy-to hdfs://srv2:8082/hbase -mappers 16 +$ bin/hbase org.apache.hadoop.hbase.fs.legacy.snapshot.ExportSnapshot -snapshot MySnapshot -copy-to hdfs://srv2:8082/hbase -mappers 16 ---- .Limiting Bandwidth Consumption @@ -2072,7 +2072,7 @@ The following example limits the above example to 200 MB/sec. [source,bourne] ---- -$ bin/hbase org.apache.hadoop.hbase.snapshot.ExportSnapshot -snapshot MySnapshot -copy-to hdfs://srv2:8082/hbase -mappers 16 -bandwidth 200 +$ bin/hbase org.apache.hadoop.hbase.fs.legacy.snapshot.ExportSnapshot -snapshot MySnapshot -copy-to hdfs://srv2:8082/hbase -mappers 16 -bandwidth 200 ---- [[snapshots_s3]] @@ -2091,12 +2091,12 @@ and `s3://` protocols have various limitations and do not use the Amazon AWS SDK the commands to export and restore the snapshot. After you have fulfilled the prerequisites, take the snapshot like you normally would. -Afterward, you can export it using the `org.apache.hadoop.hbase.snapshot.ExportSnapshot` +Afterward, you can export it using the `org.apache.hadoop.hbase.fs.legacy.snapshot.ExportSnapshot` command like the one below, substituting your own `s3a://` path in the `copy-from` or `copy-to` directive and substituting or modifying other options as required: ---- -$ hbase org.apache.hadoop.hbase.snapshot.ExportSnapshot \ +$ hbase org.apache.hadoop.hbase.fs.legacy.snapshot.ExportSnapshot \ -snapshot MySnapshot \ -copy-from hdfs://srv2:8082/hbase \ -copy-to s3a://<bucket>/<namespace>/hbase \ @@ -2107,7 +2107,7 @@ $ hbase org.apache.hadoop.hbase.snapshot.ExportSnapshot \ ---- ---- -$ hbase org.apache.hadoop.hbase.snapshot.ExportSnapshot \ +$ hbase org.apache.hadoop.hbase.fs.legacy.snapshot.ExportSnapshot \ -snapshot MySnapshot -copy-from s3a://<bucket>/<namespace>/hbase \ -copy-to hdfs://srv2:8082/hbase \
