saintstack commented on a change in pull request #1753:
URL: https://github.com/apache/hbase/pull/1753#discussion_r429346620
##########
File path:
hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/LogCleaner.java
##########
@@ -86,8 +87,9 @@ public LogCleaner(final int period, final Stoppable stopper,
Configuration conf,
@Override
protected boolean validate(Path file) {
- return AbstractFSWALProvider.validateWALFilename(file.getName())
- || MasterProcedureUtil.validateProcedureWALFilename(file.getName());
+ return AbstractFSWALProvider.validateWALFilename(file.getName()) ||
+ MasterProcedureUtil.validateProcedureWALFilename(file.getName()) ||
+ file.getName().endsWith(LocalStore.ARCHIVED_WAL_SUFFIX);
}
@Override
Review comment:
Yes. Of course.
##########
File path:
hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
##########
@@ -1550,10 +1563,8 @@ protected void stopServiceThreads() {
private void createProcedureExecutor() throws IOException {
MasterProcedureEnv procEnv = new MasterProcedureEnv(this);
- // Create cleaner thread pool
- cleanerPool = new DirScanPool(conf);
- procedureStore = new RegionProcedureStore(this, cleanerPool,
- new MasterProcedureEnv.FsUtilsLeaseRecovery(this));
+ procedureStore =
+ new RegionProcedureStore(this, localStore, new
MasterProcedureEnv.FsUtilsLeaseRecovery(this));
Review comment:
nice
##########
File path:
hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
##########
@@ -1550,10 +1563,8 @@ protected void stopServiceThreads() {
private void createProcedureExecutor() throws IOException {
MasterProcedureEnv procEnv = new MasterProcedureEnv(this);
- // Create cleaner thread pool
- cleanerPool = new DirScanPool(conf);
- procedureStore = new RegionProcedureStore(this, cleanerPool,
- new MasterProcedureEnv.FsUtilsLeaseRecovery(this));
+ procedureStore =
+ new RegionProcedureStore(this, localStore, new
MasterProcedureEnv.FsUtilsLeaseRecovery(this));
Review comment:
The passing of an already initialized localregion is what is nice.
##########
File path:
hbase-server/src/main/java/org/apache/hadoop/hbase/master/store/LocalRegion.java
##########
@@ -0,0 +1,325 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.store;
+
+import static org.apache.hadoop.hbase.HConstants.HREGION_LOGDIR_NAME;
+
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseIOException;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RegionInfoBuilder;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegion.FlushResult;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.CommonFSUtils;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
+import org.apache.hadoop.hbase.util.RecoverLeaseFSUtils;
+import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
+import org.apache.hadoop.hbase.wal.WAL;
+import org.apache.hadoop.hbase.wal.WALFactory;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import
org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
+import org.apache.hbase.thirdparty.com.google.common.math.IntMath;
+
+/**
+ * A region that stores data in a separated directory.
+ * <p/>
+ * FileSystem layout:
+ *
+ * <pre>
+ * hbase
+ * |
+ * --<region dir>
+ * |
+ * --data
+ * | |
+ * | --/<ns>/<table>/<encoded-region-name> <---- The
region data
+ * | |
+ * | --replay <---- The edits to replay
+ * |
+ * --WALs
+ * |
+ * --<master-server-name> <---- The WAL dir for active master
+ * |
+ * --<master-server-name>-dead <---- The WAL dir for dead master
+ * </pre>
+ *
+ * Notice that, you can use different root file system and WAL file system.
Then the above directory
+ * will be on two file systems, the root file system will have the data
directory while the WAL
+ * filesystem will have the WALs directory. The archived HFile will be moved
to the global HFile
+ * archived directory with the {@link LocalRegionParams#archivedWalSuffix()}
suffix. The archived
+ * WAL will be moved to the global WAL archived directory with the
+ * {@link LocalRegionParams#archivedHFileSuffix()} suffix.
+ */
[email protected]
+public final class LocalRegion {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LocalRegion.class);
+
+ private static final String REPLAY_EDITS_DIR = "recovered.wals";
+
+ private static final String DEAD_WAL_DIR_SUFFIX = "-dead";
Review comment:
ok
##########
File path:
hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/RegionProcedureStore.java
##########
@@ -688,11 +444,12 @@ public void cleanup() {
Cell cell = cells.get(0);
cells.clear();
if (cell.getValueLength() == 0) {
- region.delete(new Delete(cell.getRowArray(), cell.getRowOffset(),
cell.getRowLength()));
+ localStore.update(r -> r
+ .delete(new Delete(cell.getRowArray(), cell.getRowOffset(),
cell.getRowLength())));
}
}
} catch (IOException e) {
LOG.warn("Failed to clean up delete procedures", e);
}
}
-}
+}
Review comment:
Nice. This is cleaner w/ the passing in of the local region.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
