[ https://issues.apache.org/jira/browse/HBASE-7987?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13992540#comment-13992540 ]
Hudson commented on HBASE-7987: ------------------------------- SUCCESS: Integrated in HBase-TRUNK #5128 (See [https://builds.apache.org/job/HBase-TRUNK/5128/]) HBASE-7987 Snapshot Manifest file instead of multiple empty files (mbertozzi: rev 1593139) * /hbase/trunk/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MapReduceProtos.java * /hbase/trunk/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/SnapshotProtos.java * /hbase/trunk/hbase-protocol/src/main/protobuf/MapReduce.proto * /hbase/trunk/hbase-protocol/src/main/protobuf/Snapshot.proto * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/client/TableSnapshotScanner.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureManager.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/CopyRecoveredEditsTask.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ReferenceRegionHFilesTask.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ReferenceServerWALsTask.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotLogSplitter.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotTask.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/TableInfoCopyTask.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/TakeSnapshotUtils.java * /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java * /hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java * /hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java * /hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java * /hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java * /hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java * /hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java * /hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestCopyRecoveredEditsTask.java * /hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java * /hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java * /hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestReferenceRegionHFilesTask.java * /hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java * /hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java * /hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotLogSplitter.java * /hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotTask.java * /hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestWALReferenceTask.java > Snapshot Manifest file instead of multiple empty files > ------------------------------------------------------ > > Key: HBASE-7987 > URL: https://issues.apache.org/jira/browse/HBASE-7987 > Project: HBase > Issue Type: Improvement > Components: snapshots > Reporter: Matteo Bertozzi > Assignee: Matteo Bertozzi > Fix For: 0.99.0 > > Attachments: HBASE-7987-v0.patch, HBASE-7987-v1.patch, > HBASE-7987-v2.patch, HBASE-7987-v2.sketch, HBASE-7987-v3.patch, > HBASE-7987-v4.patch, HBASE-7987-v5.patch, HBASE-7987-v6.patch, > HBASE-7987.sketch > > > Currently taking a snapshot means creating one empty file for each file in > the source table directory, plus copying the .regioninfo file for each > region, the table descriptor file and a snapshotInfo file. > during the restore or snapshot verification we traverse the filesystem > (fs.listStatus()) to find the snapshot files, and we open the .regioninfo > files to get the information. > to avoid hammering the NameNode and having lots of empty files, we can use a > manifest file that contains the list of files and information that we need. > To keep the RS parallelism that we have, each RS can write its own manifest. > {code} > message SnapshotDescriptor { > required string name; > optional string table; > optional int64 creationTime; > optional Type type; > optional int32 version; > } > message SnapshotRegionManifest { > optional int32 version; > required RegionInfo regionInfo; > repeated FamilyFiles familyFiles; > message StoreFile { > required string name; > optional Reference reference; > } > message FamilyFiles { > required bytes familyName; > repeated StoreFile storeFiles; > } > } > {code} > {code} > /hbase/.snapshot/<snapshotName> > /hbase/.snapshot/<snapshotName>/snapshotInfo > /hbase/.snapshot/<snapshotName>/<tableName> > /hbase/.snapshot/<snapshotName>/<tableName>/tableInfo > /hbase/.snapshot/<snapshotName>/<tableName>/regionManifest(.n) > {code} -- This message was sent by Atlassian JIRA (v6.2#6252)