This is an automated email from the ASF dual-hosted git repository. bbeaudreault pushed a commit to branch branch-2.6 in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.6 by this push: new a0618499388 HBASE-28412 Select correct target table for incremental backup restore (#5776) a0618499388 is described below commit a06184993886874b4b345e2b75fb550e48632ca1 Author: Ruben Van Wanzeele <rube...@ngdata.com> AuthorDate: Tue Mar 26 13:15:00 2024 +0100 HBASE-28412 Select correct target table for incremental backup restore (#5776) Contributed-by: Ruben Van Wanzeele <rube...@ngdata.com> Signed-off-by: Bryan Beaudreault <bbeaudrea...@apache.org> --- .../backup/mapreduce/MapReduceRestoreJob.java | 4 +- .../apache/hadoop/hbase/backup/BackupTestUtil.java | 55 +++++++ ...va => TestBackupRestoreOnEmptyEnvironment.java} | 159 +++++++++------------ .../backup/TestBackupRestoreWithModifications.java | 28 +--- 4 files changed, 130 insertions(+), 116 deletions(-) diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceRestoreJob.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceRestoreJob.java index 55f6bff04cb..5d654c0d85b 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceRestoreJob.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceRestoreJob.java @@ -74,9 +74,7 @@ public class MapReduceRestoreJob implements RestoreJob { BackupUtils.getFileNameCompatibleString(newTableNames[i]), getConf()); Configuration conf = getConf(); conf.set(bulkOutputConfKey, bulkOutputPath.toString()); - String[] playerArgs = { dirs, - fullBackupRestore ? newTableNames[i].getNameAsString() : tableNames[i].getNameAsString() }; - + String[] playerArgs = { dirs, newTableNames[i].getNameAsString() }; int result; try { diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/BackupTestUtil.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/BackupTestUtil.java new file mode 100644 index 00000000000..3665eeb7a76 --- /dev/null +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/BackupTestUtil.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; +import org.apache.hadoop.hbase.backup.impl.BackupManager; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public class BackupTestUtil { + private BackupTestUtil() { + } + + static BackupInfo verifyBackup(Configuration conf, String backupId, BackupType expectedType, + BackupInfo.BackupState expectedState) throws IOException { + try (Connection connection = ConnectionFactory.createConnection(conf); + BackupAdmin backupAdmin = new BackupAdminImpl(connection)) { + BackupInfo backupInfo = backupAdmin.getBackupInfo(backupId); + + // Verify managed backup in HBase + assertEquals(backupId, backupInfo.getBackupId()); + assertEquals(expectedState, backupInfo.getState()); + assertEquals(expectedType, backupInfo.getType()); + return backupInfo; + } + } + + static void enableBackup(Configuration conf) { + // Enable backup + conf.setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true); + BackupManager.decorateMasterConfiguration(conf); + BackupManager.decorateRegionServerConfiguration(conf); + } +} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreWithModifications.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreOnEmptyEnvironment.java similarity index 63% copy from hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreWithModifications.java copy to hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreOnEmptyEnvironment.java index 2cf96bace0d..6fb538fe629 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreWithModifications.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreOnEmptyEnvironment.java @@ -18,30 +18,27 @@ package org.apache.hadoop.hbase.backup; import static org.apache.hadoop.hbase.backup.BackupInfo.BackupState.COMPLETE; +import static org.apache.hadoop.hbase.backup.BackupTestUtil.enableBackup; +import static org.apache.hadoop.hbase.backup.BackupTestUtil.verifyBackup; import static org.apache.hadoop.hbase.backup.BackupType.FULL; +import static org.apache.hadoop.hbase.backup.BackupType.INCREMENTAL; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.io.IOException; -import java.nio.ByteBuffer; import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.UUID; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; -import org.apache.hadoop.hbase.backup.impl.BackupManager; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Connection; @@ -51,13 +48,11 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; -import org.apache.hadoop.hbase.io.hfile.HFile; -import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testing.TestingHBaseCluster; import org.apache.hadoop.hbase.testing.TestingHBaseClusterOption; -import org.apache.hadoop.hbase.tool.BulkLoadHFiles; import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -71,29 +66,27 @@ import org.slf4j.LoggerFactory; @Category(MediumTests.class) @RunWith(Parameterized.class) -public class TestBackupRestoreWithModifications { +public class TestBackupRestoreOnEmptyEnvironment { private static final Logger LOG = - LoggerFactory.getLogger(TestBackupRestoreWithModifications.class); + LoggerFactory.getLogger(TestBackupRestoreOnEmptyEnvironment.class); @ClassRule public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestBackupRestoreWithModifications.class); + HBaseClassTestRule.forClass(TestBackupRestoreOnEmptyEnvironment.class); - @Parameterized.Parameters(name = "{index}: useBulkLoad={0}") + @Parameterized.Parameters(name = "{index}: restoreToOtherTable={0}") public static Iterable<Object[]> data() { return HBaseCommonTestingUtility.BOOLEAN_PARAMETERIZED; } @Parameterized.Parameter(0) - public boolean useBulkLoad; - + public boolean restoreToOtherTable; private TableName sourceTable; private TableName targetTable; - private List<TableName> allTables; private static TestingHBaseCluster cluster; - private static final Path BACKUP_ROOT_DIR = new Path("backupIT"); + private static Path BACKUP_ROOT_DIR; private static final byte[] COLUMN_FAMILY = Bytes.toBytes("0"); @BeforeClass @@ -102,6 +95,7 @@ public class TestBackupRestoreWithModifications { enableBackup(conf); cluster = TestingHBaseCluster.create(TestingHBaseClusterOption.builder().conf(conf).build()); cluster.start(); + BACKUP_ROOT_DIR = new Path(new Path(conf.get("fs.defaultFS")), new Path("/backupIT")); } @AfterClass @@ -111,38 +105,68 @@ public class TestBackupRestoreWithModifications { @Before public void setUp() throws Exception { - sourceTable = TableName.valueOf("table-" + useBulkLoad); - targetTable = TableName.valueOf("another-table-" + useBulkLoad); - allTables = Arrays.asList(sourceTable, targetTable); + sourceTable = TableName.valueOf("table"); + targetTable = TableName.valueOf("another-table"); createTable(sourceTable); createTable(targetTable); } + @After + public void removeTables() throws Exception { + deleteTables(); + } + @Test - public void testModificationsOnTable() throws Exception { - Instant timestamp = Instant.now(); + public void testRestoreToCorrectTable() throws Exception { + Instant timestamp = Instant.now().minusSeconds(10); // load some data - load(sourceTable, timestamp, "data"); + putLoad(sourceTable, timestamp, "data"); - String backupId = backup(FULL, allTables); - BackupInfo backupInfo = verifyBackup(backupId, FULL, COMPLETE); + String backupId = backup(FULL, Collections.singletonList(sourceTable)); + BackupInfo backupInfo = verifyBackup(cluster.getConf(), backupId, FULL, COMPLETE); assertTrue(backupInfo.getTables().contains(sourceTable)); - restore(backupId, sourceTable, targetTable); - validateDataEquals(sourceTable, "data"); - validateDataEquals(targetTable, "data"); + LOG.info("Deleting the tables before restore ..."); + deleteTables(); + + if (restoreToOtherTable) { + restore(backupId, sourceTable, targetTable); + validateDataEquals(targetTable, "data"); + } else { + restore(backupId, sourceTable, sourceTable); + validateDataEquals(sourceTable, "data"); + } - // load new data on the same timestamp - load(sourceTable, timestamp, "changed_data"); + } - backupId = backup(FULL, allTables); - backupInfo = verifyBackup(backupId, FULL, COMPLETE); - assertTrue(backupInfo.getTables().contains(sourceTable)); + @Test + public void testRestoreCorrectTableForIncremental() throws Exception { + Instant timestamp = Instant.now().minusSeconds(10); + + // load some data + putLoad(sourceTable, timestamp, "data"); + + String backupId = backup(FULL, Collections.singletonList(sourceTable)); + verifyBackup(cluster.getConf(), backupId, FULL, COMPLETE); + + // some incremental data + putLoad(sourceTable, timestamp.plusMillis(1), "new_data"); + + String backupId2 = backup(INCREMENTAL, Collections.singletonList(sourceTable)); + verifyBackup(cluster.getConf(), backupId2, INCREMENTAL, COMPLETE); + + LOG.info("Deleting the tables before restore ..."); + deleteTables(); + + if (restoreToOtherTable) { + restore(backupId2, sourceTable, targetTable); + validateDataEquals(targetTable, "new_data"); + } else { + restore(backupId2, sourceTable, sourceTable); + validateDataEquals(sourceTable, "new_data"); + } - restore(backupId, sourceTable, targetTable); - validateDataEquals(sourceTable, "changed_data"); - validateDataEquals(targetTable, "changed_data"); } private void createTable(TableName tableName) throws IOException { @@ -154,11 +178,15 @@ public class TestBackupRestoreWithModifications { } } - private void load(TableName tableName, Instant timestamp, String data) throws IOException { - if (useBulkLoad) { - hFileBulkLoad(tableName, timestamp, data); - } else { - putLoad(tableName, timestamp, data); + private void deleteTables() throws IOException { + try (Connection connection = ConnectionFactory.createConnection(cluster.getConf()); + Admin admin = connection.getAdmin()) { + for (TableName table : Arrays.asList(sourceTable, targetTable)) { + if (admin.tableExists(table)) { + admin.disableTable(table); + admin.deleteTable(table); + } + } } } @@ -184,30 +212,6 @@ public class TestBackupRestoreWithModifications { } } - private void hFileBulkLoad(TableName tableName, Instant timestamp, String data) - throws IOException { - FileSystem fs = FileSystem.get(cluster.getConf()); - LOG.info("Writing new data to HBase using BulkLoad: {}", data); - // HFiles require this strict directory structure to allow to load them - Path hFileRootPath = new Path("/tmp/hfiles_" + UUID.randomUUID()); - fs.mkdirs(hFileRootPath); - Path hFileFamilyPath = new Path(hFileRootPath, Bytes.toString(COLUMN_FAMILY)); - fs.mkdirs(hFileFamilyPath); - try (HFile.Writer writer = HFile.getWriterFactoryNoCache(cluster.getConf()) - .withPath(fs, new Path(hFileFamilyPath, "hfile_" + UUID.randomUUID())) - .withFileContext(new HFileContextBuilder().withTableName(tableName.toBytes()) - .withColumnFamily(COLUMN_FAMILY).build()) - .create()) { - for (int i = 0; i < 10; i++) { - writer.append(new KeyValue(Bytes.toBytes(i), COLUMN_FAMILY, Bytes.toBytes("data"), - timestamp.toEpochMilli(), Bytes.toBytes(data))); - } - } - Map<BulkLoadHFiles.LoadQueueItem, ByteBuffer> result = - BulkLoadHFiles.create(cluster.getConf()).bulkLoad(tableName, hFileRootPath); - assertFalse(result.isEmpty()); - } - private String backup(BackupType backupType, List<TableName> tables) throws IOException { LOG.info("Creating the backup ..."); @@ -238,7 +242,6 @@ public class TestBackupRestoreWithModifications { try (Connection connection = ConnectionFactory.createConnection(cluster.getConf()); Table table = connection.getTable(tableName)) { Scan scan = new Scan(); - scan.readAllVersions(); scan.setRaw(true); scan.setBatch(100); @@ -251,26 +254,4 @@ public class TestBackupRestoreWithModifications { } } } - - private BackupInfo verifyBackup(String backupId, BackupType expectedType, - BackupInfo.BackupState expectedState) throws IOException { - try (Connection connection = ConnectionFactory.createConnection(cluster.getConf()); - BackupAdmin backupAdmin = new BackupAdminImpl(connection)) { - BackupInfo backupInfo = backupAdmin.getBackupInfo(backupId); - - // Verify managed backup in HBase - assertEquals(backupId, backupInfo.getBackupId()); - assertEquals(expectedState, backupInfo.getState()); - assertEquals(expectedType, backupInfo.getType()); - return backupInfo; - } - } - - private static void enableBackup(Configuration conf) { - // Enable backup - conf.setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true); - BackupManager.decorateMasterConfiguration(conf); - BackupManager.decorateRegionServerConfiguration(conf); - } - } diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreWithModifications.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreWithModifications.java index 2cf96bace0d..72c4a464e94 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreWithModifications.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreWithModifications.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hbase.backup; import static org.apache.hadoop.hbase.backup.BackupInfo.BackupState.COMPLETE; +import static org.apache.hadoop.hbase.backup.BackupTestUtil.enableBackup; +import static org.apache.hadoop.hbase.backup.BackupTestUtil.verifyBackup; import static org.apache.hadoop.hbase.backup.BackupType.FULL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -41,7 +43,6 @@ import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; -import org.apache.hadoop.hbase.backup.impl.BackupManager; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Connection; @@ -126,7 +127,7 @@ public class TestBackupRestoreWithModifications { load(sourceTable, timestamp, "data"); String backupId = backup(FULL, allTables); - BackupInfo backupInfo = verifyBackup(backupId, FULL, COMPLETE); + BackupInfo backupInfo = verifyBackup(cluster.getConf(), backupId, FULL, COMPLETE); assertTrue(backupInfo.getTables().contains(sourceTable)); restore(backupId, sourceTable, targetTable); @@ -137,7 +138,7 @@ public class TestBackupRestoreWithModifications { load(sourceTable, timestamp, "changed_data"); backupId = backup(FULL, allTables); - backupInfo = verifyBackup(backupId, FULL, COMPLETE); + backupInfo = verifyBackup(cluster.getConf(), backupId, FULL, COMPLETE); assertTrue(backupInfo.getTables().contains(sourceTable)); restore(backupId, sourceTable, targetTable); @@ -252,25 +253,4 @@ public class TestBackupRestoreWithModifications { } } - private BackupInfo verifyBackup(String backupId, BackupType expectedType, - BackupInfo.BackupState expectedState) throws IOException { - try (Connection connection = ConnectionFactory.createConnection(cluster.getConf()); - BackupAdmin backupAdmin = new BackupAdminImpl(connection)) { - BackupInfo backupInfo = backupAdmin.getBackupInfo(backupId); - - // Verify managed backup in HBase - assertEquals(backupId, backupInfo.getBackupId()); - assertEquals(expectedState, backupInfo.getState()); - assertEquals(expectedType, backupInfo.getType()); - return backupInfo; - } - } - - private static void enableBackup(Configuration conf) { - // Enable backup - conf.setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true); - BackupManager.decorateMasterConfiguration(conf); - BackupManager.decorateRegionServerConfiguration(conf); - } - }