anmolnar commented on code in PR #6518: URL: https://github.com/apache/hbase/pull/6518#discussion_r1870112302
########## hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/replication/BackupFileSystemManager.java: ########## @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup.replication; + +import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + [email protected] +public class BackupFileSystemManager { + private static final Logger LOG = LoggerFactory.getLogger(BackupFileSystemManager.class); + + public static final String WALS_DIR = "WALs"; + public static final String BULKLOAD_FILES_DIR = "bulk-load-files"; + private final String peerId; + private final FileSystem backupFs; + private final Path backupRootDir; + private Path walsDir; + private Path bulkLoadFilesDir; + + public BackupFileSystemManager(String peerId, Configuration conf, String backupRootDirStr) + throws IOException { + this.peerId = peerId; + this.backupRootDir = new Path(backupRootDirStr); + this.backupFs = FileSystem.get(backupRootDir.toUri(), conf); + initBackupDirectories(); + } + + private void initBackupDirectories() throws IOException { + LOG.info("{} Initializing backup directories under root: {}", Utils.logPeerId(peerId), + backupRootDir); + try { + walsDir = createDirectoryIfNotExists(WALS_DIR); + bulkLoadFilesDir = createDirectoryIfNotExists(BULKLOAD_FILES_DIR); + } catch (IOException e) { + LOG.error("{} Failed to initialize backup directories: {}", Utils.logPeerId(peerId), + e.getMessage(), e); + throw e; + } + } + + private Path createDirectoryIfNotExists(String dirName) throws IOException { + Path dirPath = new Path(backupRootDir, dirName); + if (backupFs.exists(dirPath)) { + LOG.info("{} Directory already exists: {}", Utils.logPeerId(peerId), dirPath); + } else { + backupFs.mkdirs(dirPath); + LOG.info("{} Successfully created directory: {}", Utils.logPeerId(peerId), dirPath); + } Review Comment: How many HBase instances are running this code? Isn't there a race condition here? Is this run by master only? ########## hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/replication/ContinuousBackupManager.java: ########## @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup.replication; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.wal.WAL; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Manages the continuous backup process for HBase WAL entries and bulk load files. + * <p> + * This class is responsible for initializing backup components, processing WAL entries, staging + * them for backup, and committing the backup to the configured storage. It uses + * {@link BackupFileSystemManager} for handling file system operations and + * {@link ContinuousBackupStagingManager} for managing staging. + * </p> + */ [email protected] +public class ContinuousBackupManager { + private static final Logger LOG = LoggerFactory.getLogger(ContinuousBackupManager.class); + public static final String CONF_BACKUP_ROOT_DIR = "hbase.backup.root.dir"; Review Comment: Why do you need this root dir configured? New full backup command will get a fully qualified backup path: ``` hbase backup create full s3a://my-hbase-bucket/backups/backup_0001 --continous ``` Is this for the staging area? ########## hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/replication/ContinuousBackupManager.java: ########## @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup.replication; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.wal.WAL; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Manages the continuous backup process for HBase WAL entries and bulk load files. + * <p> + * This class is responsible for initializing backup components, processing WAL entries, staging + * them for backup, and committing the backup to the configured storage. It uses + * {@link BackupFileSystemManager} for handling file system operations and + * {@link ContinuousBackupStagingManager} for managing staging. + * </p> + */ [email protected] +public class ContinuousBackupManager { + private static final Logger LOG = LoggerFactory.getLogger(ContinuousBackupManager.class); + public static final String CONF_BACKUP_ROOT_DIR = "hbase.backup.root.dir"; + public static final String CONF_BACKUP_MAX_WAL_SIZE = "hbase.backup.max.wal.size"; + public static final long DEFAULT_MAX_WAL_SIZE = 128 * 1024 * 1024; Review Comment: nit: add short comment: `// 128 MB` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
