This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new e22a1936a77 HBASE-29797 Should not create wal directory when creating
WAL instance (#7585)
e22a1936a77 is described below
commit e22a1936a7711d768b5c29bc44c55ebf74e03f71
Author: Duo Zhang <[email protected]>
AuthorDate: Mon Jan 5 23:01:47 2026 +0800
HBASE-29797 Should not create wal directory when creating WAL instance
(#7585)
Signed-off-by: Charles Connell <[email protected]>
(cherry picked from commit add6f22a6e417b7533edf4543172bd01ea37cd44)
---
hbase-server/pom.xml | 5 ++
.../hadoop/hbase/regionserver/HRegionServer.java | 4 +-
.../hbase/regionserver/wal/AbstractFSWAL.java | 6 +-
.../org/apache/hadoop/hbase/wal/WALFactory.java | 34 ++++++---
.../org/apache/hadoop/hbase/HBaseTestingUtil.java | 5 +-
.../apache/hadoop/hbase/master/TestWALFencing.java | 81 ++++++++++++++++++++++
.../hbase/regionserver/TestCompactionPolicy.java | 1 +
.../regionserver/TestFailedAppendAndSync.java | 1 +
.../hadoop/hbase/regionserver/TestHRegion.java | 22 +++---
.../hadoop/hbase/regionserver/TestLogRoller.java | 5 +-
.../hbase/regionserver/wal/AbstractTestFSWAL.java | 7 +-
.../regionserver/wal/AbstractTestWALReplay.java | 1 +
.../regionserver/wal/TestAsyncFSWALDurability.java | 2 +-
.../regionserver/wal/TestAsyncFSWALRollStuck.java | 1 +
.../hbase/regionserver/wal/TestAsyncWALReplay.java | 8 ++-
.../hadoop/hbase/regionserver/wal/TestFSHLog.java | 2 +
.../regionserver/wal/TestFSHLogDurability.java | 2 +-
.../hbase/regionserver/wal/TestWALReplay.java | 4 +-
.../regionserver/wal/WALDurabilityTestBase.java | 8 ++-
.../apache/hadoop/hbase/wal/TestWALFactory.java | 27 ++++----
.../hadoop/hbase/wal/TestWALSplitToHFile.java | 5 +-
pom.xml | 7 ++
22 files changed, 186 insertions(+), 52 deletions(-)
diff --git a/hbase-server/pom.xml b/hbase-server/pom.xml
index 295bb27f16b..2593d7499ec 100644
--- a/hbase-server/pom.xml
+++ b/hbase-server/pom.xml
@@ -296,6 +296,11 @@
<artifactId>junit-vintage-engine</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.awaitility</groupId>
+ <artifactId>awaitility</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 39fe6322ed5..eea82ca511e 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1799,8 +1799,8 @@ public class HRegionServer extends
HBaseServerBase<RSRpcServices>
throw new RegionServerRunningException(
"Region server has already created directory at " +
this.serverName.toString());
}
- // Always create wal directory as now we need this when master restarts to
find out the live
- // region servers.
+ // Create wal directory here and we will never create it again in other
places. This is
+ // important to make sure that our fencing way takes effect. See
HBASE-29797 for more details.
if (!this.walFs.mkdirs(logDir)) {
throw new IOException("Can not create wal directory " + logDir);
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
index 77c296b096c..1055765b3ef 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
@@ -536,10 +536,8 @@ public abstract class AbstractFSWAL<W extends WriterBase>
implements WAL {
this.remoteFs = remoteFs;
this.remoteWALDir = remoteWALDir;
- if (!fs.exists(walDir) && !fs.mkdirs(walDir)) {
- throw new IOException("Unable to mkdir " + walDir);
- }
-
+ // Here we only crate archive dir, without wal dir. This is to make sure
that our fencing way
+ // takes effect. See HBASE-29797 for more details.
if (!fs.exists(this.walArchiveDir)) {
if (!fs.mkdirs(this.walArchiveDir)) {
throw new IOException("Unable to mkdir " + this.walArchiveDir);
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java
index 6b638cdda7f..89e713ccb2a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java
@@ -36,6 +36,7 @@ import
org.apache.hadoop.hbase.regionserver.wal.ProtobufWALStreamReader;
import org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.util.CancelableProgressable;
+import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.LeaseNotRecoveredException;
import org.apache.hadoop.hbase.wal.WALProvider.Writer;
@@ -211,7 +212,7 @@ public class WALFactory {
public WALFactory(Configuration conf, String factoryId) throws IOException {
// default enableSyncReplicationWALProvider is true, only disable
SyncReplicationWALProvider
// for HMaster or HRegionServer which take system table only. See
HBASE-19999
- this(conf, factoryId, null);
+ this(conf, factoryId, null, true);
}
/**
@@ -228,17 +229,30 @@ public class WALFactory {
*/
public WALFactory(Configuration conf, ServerName serverName, Abortable
abortable)
throws IOException {
- this(conf, serverName.toString(), abortable);
+ this(conf, serverName.toString(), abortable, false);
+ }
+
+ private static void createWALDirectory(Configuration conf, String factoryId)
throws IOException {
+ FileSystem walFs = CommonFSUtils.getWALFileSystem(conf);
+ Path walRootDir = CommonFSUtils.getWALRootDir(conf);
+ Path walDir = new Path(walRootDir,
AbstractFSWALProvider.getWALDirectoryName(factoryId));
+ if (!walFs.exists(walDir) && !walFs.mkdirs(walDir)) {
+ throw new IOException("Can not create wal directory " + walDir);
+ }
}
/**
- * @param conf must not be null, will keep a reference to read params
in later reader/writer
- * instances.
- * @param factoryId a unique identifier for this factory. used i.e. by
filesystem implementations
- * to make a directory
- * @param abortable the server associated with this WAL file
+ * @param conf must not be null, will keep a reference to read
params in later
+ * reader/writer instances.
+ * @param factoryId a unique identifier for this factory. used i.e.
by filesystem
+ * implementations to make a directory
+ * @param abortable the server associated with this WAL file
+ * @param createWalDirectory pass {@code true} for testing purpose, to
create the wal directory
+ * automatically. In normal code path, we should
create it in
+ * HRegionServer setup.
*/
- private WALFactory(Configuration conf, String factoryId, Abortable
abortable) throws IOException {
+ private WALFactory(Configuration conf, String factoryId, Abortable abortable,
+ boolean createWalDirectory) throws IOException {
// until we've moved reader/writer construction down into providers, this
initialization must
// happen prior to provider initialization, in case they need to
instantiate a reader/writer.
timeoutMillis = conf.getInt("hbase.hlog.open.timeout", 300000);
@@ -259,6 +273,10 @@ public class WALFactory {
REPLICATION_WAL_PROVIDER, this.abortable);
// end required early initialization
if (conf.getBoolean(WAL_ENABLED, true)) {
+ if (createWalDirectory) {
+ // for testing only
+ createWALDirectory(conf, factoryId);
+ }
WALProvider provider = createProvider(getProviderClass(WAL_PROVIDER,
DEFAULT_WAL_PROVIDER));
provider.init(this, conf, null, this.abortable);
provider.addWALActionsListener(new MetricsWAL());
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java
index 3be8776d652..759d3ea4955 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java
@@ -2194,8 +2194,9 @@ public class HBaseTestingUtil extends HBaseZKTestingUtil {
// The WAL subsystem will use the default rootDir rather than the passed
in rootDir
// unless I pass along via the conf.
Configuration confForWAL = new Configuration(conf);
- confForWAL.set(HConstants.HBASE_DIR, rootDir.toString());
- return new WALFactory(confForWAL, "hregion-" +
RandomStringUtils.randomNumeric(8)).getWAL(hri);
+ CommonFSUtils.setRootDir(confForWAL, rootDir);
+ return new WALFactory(confForWAL, "hregion-" +
RandomStringUtils.insecure().nextNumeric(8))
+ .getWAL(hri);
}
/**
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestWALFencing.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestWALFencing.java
new file mode 100644
index 00000000000..d77aa67091c
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestWALFencing.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import static org.awaitility.Awaitility.await;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtil;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.RecoverLeaseFSUtils;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Testcase for HBASE-29797, where the lazy initialized WALProvider may
recreate the WAL directory
+ * and cause our fencing way loses efficacy.
+ */
+@Tag(MasterTests.TAG)
+@Tag(MediumTests.TAG)
+public class TestWALFencing {
+
+ private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
+
+ @BeforeAll
+ public static void setUp() throws Exception {
+ UTIL.startMiniCluster(3);
+ UTIL.getAdmin().balancerSwitch(false, true);
+ }
+
+ @AfterAll
+ public static void tearDown() throws IOException {
+ UTIL.shutdownMiniCluster();
+ }
+
+ @Test
+ public void testMoveMeta() throws Exception {
+ HRegionServer metaRs =
UTIL.getRSForFirstRegionInTable(TableName.META_TABLE_NAME);
+ HRegionServer otherRs = UTIL.getOtherRegionServer(metaRs);
+ // do fencing here, i.e, kill otherRs
+ Path splittingDir =
UTIL.getMiniHBaseCluster().getMaster().getMasterWalManager()
+ .getLogDirs(Collections.singleton(otherRs.getServerName())).get(0);
+ for (FileStatus walFile :
otherRs.getWALFileSystem().listStatus(splittingDir)) {
+ RecoverLeaseFSUtils.recoverFileLease(otherRs.getWALFileSystem(),
walFile.getPath(),
+ otherRs.getConfiguration());
+ }
+ // move meta region to otherRs, which should fail and crash otherRs, and
then master will try to
+ // assign meta region to another rs
+ RegionInfo metaRegionInfo =
metaRs.getRegions().stream().map(Region::getRegionInfo)
+ .filter(RegionInfo::isMetaRegion).findAny().get();
+ UTIL.getAdmin().move(metaRegionInfo.getRegionName(),
otherRs.getServerName());
+ // make sure that meta region is not on otherRs
+ await().during(Duration.ofSeconds(5)).atMost(Duration.ofSeconds(6))
+ .until(() -> otherRs.getRegions(TableName.META_TABLE_NAME).isEmpty());
+ }
+}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionPolicy.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionPolicy.java
index 16987fc2fce..05c58fde309 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionPolicy.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionPolicy.java
@@ -104,6 +104,7 @@ public class TestCompactionPolicy {
.setColumnFamily(familyDescriptor).build();
RegionInfo info =
RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
+ fs.mkdirs(new Path(basedir, logName));
hlog = new FSHLog(fs, basedir, logName, conf);
hlog.init();
ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0,
null,
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFailedAppendAndSync.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFailedAppendAndSync.java
index 9a946992b77..c6cf1393978 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFailedAppendAndSync.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFailedAppendAndSync.java
@@ -197,6 +197,7 @@ public class TestFailedAppendAndSync {
// the test.
FileSystem fs = FileSystem.get(CONF);
Path rootDir = new Path(dir + getName());
+ fs.mkdirs(new Path(rootDir, getName()));
DodgyFSLog dodgyWAL = new DodgyFSLog(fs, (Server) services, rootDir,
getName(), CONF);
dodgyWAL.init();
LogRoller logRoller = new LogRoller(services);
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index 4dc92c2d5e5..aabbf739b12 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -370,8 +370,9 @@ public class TestHRegion {
}
FileSystem fs = FileSystem.get(CONF);
- Path rootDir = new Path(dir + "testMemstoreSnapshotSize");
- MyFaultyFSLog faultyLog = new MyFaultyFSLog(fs, rootDir,
"testMemstoreSnapshotSize", CONF);
+ Path rootDir = new Path(dir + method);
+ fs.mkdirs(new Path(rootDir, method));
+ MyFaultyFSLog faultyLog = new MyFaultyFSLog(fs, rootDir, method, CONF);
faultyLog.init();
region = initHRegion(tableName, null, null, CONF, false,
Durability.SYNC_WAL, faultyLog,
COLUMN_FAMILY_BYTES);
@@ -413,10 +414,10 @@ public class TestHRegion {
@Test
public void testMemstoreSizeAccountingWithFailedPostBatchMutate() throws
IOException {
- String testName = "testMemstoreSizeAccountingWithFailedPostBatchMutate";
FileSystem fs = FileSystem.get(CONF);
- Path rootDir = new Path(dir + testName);
- FSHLog hLog = new FSHLog(fs, rootDir, testName, CONF);
+ Path rootDir = new Path(dir + method);
+ fs.mkdirs(new Path(rootDir, method));
+ FSHLog hLog = new FSHLog(fs, rootDir, method, CONF);
hLog.init();
region = initHRegion(tableName, null, null, CONF, false,
Durability.SYNC_WAL, hLog,
COLUMN_FAMILY_BYTES);
@@ -1252,8 +1253,10 @@ public class TestHRegion {
};
}
}
- FailAppendFlushMarkerWAL wal = new
FailAppendFlushMarkerWAL(FileSystem.get(walConf),
- CommonFSUtils.getRootDir(walConf), method, walConf);
+ FileSystem fs = FileSystem.get(walConf);
+ Path rootDir = CommonFSUtils.getRootDir(walConf);
+ fs.mkdirs(new Path(rootDir, method));
+ FailAppendFlushMarkerWAL wal = new FailAppendFlushMarkerWAL(fs, rootDir,
method, walConf);
wal.init();
this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW,
HConstants.EMPTY_END_ROW, CONF,
false, Durability.USE_DEFAULT, wal, family);
@@ -3354,8 +3357,9 @@ public class TestHRegion {
@Test
public void testDataInMemoryWithoutWAL() throws IOException {
FileSystem fs = FileSystem.get(CONF);
- Path rootDir = new Path(dir + "testDataInMemoryWithoutWAL");
- FSHLog hLog = new FSHLog(fs, rootDir, "testDataInMemoryWithoutWAL", CONF);
+ Path rootDir = new Path(dir + method);
+ fs.mkdirs(new Path(rootDir, method));
+ FSHLog hLog = new FSHLog(fs, rootDir, method, CONF);
hLog.init();
// This chunk creation is done throughout the code base. Do we want to
move it into core?
// It is missing from this test. W/o it we NPE.
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestLogRoller.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestLogRoller.java
index 34d2567805f..70b0391f02b 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestLogRoller.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestLogRoller.java
@@ -61,12 +61,13 @@ public class TestLogRoller {
private static FileSystem FS;
@Before
- public void setup() throws Exception {
+ public void setUp() throws Exception {
CONF = TEST_UTIL.getConfiguration();
CONF.setInt("hbase.regionserver.logroll.period", LOG_ROLL_PERIOD);
CONF.setInt(HConstants.THREAD_WAKE_FREQUENCY, 300);
ROOT_DIR = TEST_UTIL.getRandomDir();
FS = FileSystem.get(CONF);
+ FS.mkdirs(new Path(ROOT_DIR, LOG_DIR));
RegionServerServices services = Mockito.mock(RegionServerServices.class);
Mockito.when(services.getConfiguration()).thenReturn(CONF);
ROLLER = new LogRoller(services);
@@ -77,7 +78,7 @@ public class TestLogRoller {
public void tearDown() throws Exception {
ROLLER.close();
FS.close();
- TEST_UTIL.shutdownMiniCluster();
+ TEST_UTIL.cleanupTestDir();
}
@Test
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java
index e8a364cd54c..f26c3f3e661 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java
@@ -121,6 +121,7 @@ public abstract class AbstractTestFSWAL {
final Path hbaseWALDir = TEST_UTIL.createWALRootDir();
DIR = new Path(hbaseWALDir, currentTest.getMethodName());
assertNotEquals(hbaseDir, hbaseWALDir);
+ FS.mkdirs(DIR);
}
@BeforeClass
@@ -393,9 +394,8 @@ public abstract class AbstractTestFSWAL {
@Test(expected = IOException.class)
public void testFailedToCreateWALIfParentRenamed()
throws IOException, CommonFSUtils.StreamLacksCapabilityException {
- final String name = "testFailedToCreateWALIfParentRenamed";
- AbstractFSWAL<?> wal = newWAL(FS, CommonFSUtils.getWALRootDir(CONF), name,
- HConstants.HREGION_OLDLOGDIR_NAME, CONF, null, true, null, null);
+ AbstractFSWAL<?> wal = newWAL(FS, CommonFSUtils.getWALRootDir(CONF),
+ currentTest.getMethodName(), HConstants.HREGION_OLDLOGDIR_NAME, CONF,
null, true, null, null);
long filenum = EnvironmentEdgeManager.currentTime();
Path path = wal.computeFilename(filenum);
wal.createWriterInstance(FS, path);
@@ -544,6 +544,7 @@ public abstract class AbstractTestFSWAL {
private AbstractFSWAL<?> createHoldingWAL(String testName, AtomicBoolean
startHoldingForAppend,
CountDownLatch holdAppend) throws IOException {
+ FS.mkdirs(new Path(CommonFSUtils.getRootDir(CONF), testName));
AbstractFSWAL<?> wal = newWAL(FS, CommonFSUtils.getRootDir(CONF), testName,
HConstants.HREGION_OLDLOGDIR_NAME, CONF, null, true, null, null);
// newWAL has already called wal.init()
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
index 140b3184908..0c205587091 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
@@ -1047,6 +1047,7 @@ public abstract class AbstractTestWALReplay {
}
private MockWAL createMockWAL() throws IOException {
+ fs.mkdirs(new Path(hbaseRootDir, logName));
MockWAL wal = new MockWAL(fs, hbaseRootDir, logName, conf);
wal.init();
// Set down maximum recovery so we dfsclient doesn't linger retrying
something
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncFSWALDurability.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncFSWALDurability.java
index 1d1ffcdac3f..cb48b903537 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncFSWALDurability.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncFSWALDurability.java
@@ -58,7 +58,7 @@ public class TestAsyncFSWALDurability extends
WALDurabilityTestBase<CustomAsyncF
}
@Override
- protected CustomAsyncFSWAL getWAL(FileSystem fs, Path root, String logDir,
Configuration conf)
+ protected CustomAsyncFSWAL getWAL0(FileSystem fs, Path root, String logDir,
Configuration conf)
throws IOException {
CustomAsyncFSWAL wal =
new CustomAsyncFSWAL(fs, root, logDir, conf, GROUP,
NioSocketChannel.class);
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncFSWALRollStuck.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncFSWALRollStuck.java
index 931362832ed..b065b92e5d0 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncFSWALRollStuck.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncFSWALRollStuck.java
@@ -164,6 +164,7 @@ public class TestAsyncFSWALRollStuck {
}
};
+ UTIL.getTestFileSystem().mkdirs(new Path(rootDir, "log"));
WAL = new AsyncFSWAL(UTIL.getTestFileSystem(), null, rootDir, "log",
"oldlog", conf,
Arrays.asList(listener), true, null, null, null, null, EVENT_LOOP_GROUP,
CHANNEL_CLASS,
StreamSlowMonitor.create(conf, "monitor"));
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncWALReplay.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncWALReplay.java
index 7427c4d8363..5a1816859cd 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncWALReplay.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncWALReplay.java
@@ -70,9 +70,11 @@ public class TestAsyncWALReplay extends
AbstractTestWALReplay {
@Override
protected WAL createWAL(Configuration c, Path hbaseRootDir, String logName)
throws IOException {
- AsyncFSWAL wal = new AsyncFSWAL(FileSystem.get(c), null, hbaseRootDir,
logName,
- HConstants.HREGION_OLDLOGDIR_NAME, c, null, true, null, null, null,
null, GROUP,
- CHANNEL_CLASS, StreamSlowMonitor.create(c, "monitor"));
+ FileSystem fs = hbaseRootDir.getFileSystem(c);
+ fs.mkdirs(new Path(hbaseRootDir, logName));
+ AsyncFSWAL wal =
+ new AsyncFSWAL(fs, null, hbaseRootDir, logName,
HConstants.HREGION_OLDLOGDIR_NAME, c, null,
+ true, null, null, null, null, GROUP, CHANNEL_CLASS,
StreamSlowMonitor.create(c, "monitor"));
wal.init();
return wal;
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
index ec993b89768..d99b3d958cd 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
@@ -103,6 +103,7 @@ public class TestFSHLog extends AbstractTestFSWAL {
public void testSyncRunnerIndexOverflow() throws IOException,
NoSuchFieldException,
SecurityException, IllegalArgumentException, IllegalAccessException {
final String name = this.name.getMethodName();
+ FS.mkdirs(new Path(CommonFSUtils.getRootDir(CONF), name));
FSHLog log = new FSHLog(FS, CommonFSUtils.getRootDir(CONF), name,
HConstants.HREGION_OLDLOGDIR_NAME, CONF, null, true, null, null);
log.init();
@@ -140,6 +141,7 @@ public class TestFSHLog extends AbstractTestFSWAL {
final CountDownLatch flushFinished = new CountDownLatch(1);
final CountDownLatch putFinished = new CountDownLatch(1);
+ FS.mkdirs(new Path(CommonFSUtils.getRootDir(CONF), name));
try (FSHLog log = new FSHLog(FS, CommonFSUtils.getRootDir(CONF), name,
HConstants.HREGION_OLDLOGDIR_NAME, CONF, null, true, null, null)) {
log.init();
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLogDurability.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLogDurability.java
index 926092663d4..d6ca9a1ab23 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLogDurability.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLogDurability.java
@@ -36,7 +36,7 @@ public class TestFSHLogDurability extends
WALDurabilityTestBase<CustomFSHLog> {
HBaseClassTestRule.forClass(TestFSHLogDurability.class);
@Override
- protected CustomFSHLog getWAL(FileSystem fs, Path root, String logDir,
Configuration conf)
+ protected CustomFSHLog getWAL0(FileSystem fs, Path root, String logDir,
Configuration conf)
throws IOException {
CustomFSHLog wal = new CustomFSHLog(fs, root, logDir, conf);
wal.init();
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
index c0bf4e1bc95..a41b83fb41c 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
@@ -47,7 +47,9 @@ public class TestWALReplay extends AbstractTestWALReplay {
@Override
protected WAL createWAL(Configuration c, Path hbaseRootDir, String logName)
throws IOException {
- FSHLog wal = new FSHLog(FileSystem.get(c), hbaseRootDir, logName, c);
+ FileSystem fs = hbaseRootDir.getFileSystem(c);
+ fs.mkdirs(new Path(hbaseRootDir, logName));
+ FSHLog wal = new FSHLog(fs, hbaseRootDir, logName, c);
wal.init();
// Set down maximum recovery so we dfsclient doesn't linger retrying
something
// long gone.
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/WALDurabilityTestBase.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/WALDurabilityTestBase.java
index f83b7792798..d0aebad90fc 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/WALDurabilityTestBase.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/WALDurabilityTestBase.java
@@ -69,7 +69,13 @@ public abstract class WALDurabilityTestBase<T extends WAL> {
TEST_UTIL.cleanupTestDir();
}
- protected abstract T getWAL(FileSystem fs, Path root, String logDir,
Configuration conf)
+ protected final T getWAL(FileSystem fs, Path root, String logDir,
Configuration conf)
+ throws IOException {
+ fs.mkdirs(new Path(root, logDir));
+ return getWAL0(fs, root, logDir, conf);
+ }
+
+ protected abstract T getWAL0(FileSystem fs, Path root, String logDir,
Configuration conf)
throws IOException;
protected abstract void resetSyncFlag(T wal);
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java
index 2c994f091f3..9558ae60246 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java
@@ -623,7 +623,7 @@ public class TestWALFactory {
@Test
public void testWALProviders() throws IOException {
Configuration conf = new Configuration();
- WALFactory walFactory = new WALFactory(conf,
this.currentServername.toString());
+ WALFactory walFactory = new WALFactory(conf, this.currentServername, null);
assertEquals(walFactory.getWALProvider().getClass(),
walFactory.getMetaProvider().getClass());
// if providers are not set and do not enable SyncReplicationWALProvider
@@ -635,7 +635,7 @@ public class TestWALFactory {
public void testOnlySetWALProvider() throws IOException {
Configuration conf = new Configuration();
conf.set(WAL_PROVIDER, WALFactory.Providers.multiwal.name());
- WALFactory walFactory = new WALFactory(conf,
this.currentServername.toString());
+ WALFactory walFactory = new WALFactory(conf, this.currentServername, null);
// class of WALProvider and metaWALProvider are the same when
metaWALProvider is not set
assertEquals(WALFactory.Providers.multiwal.clazz,
walFactory.getWALProvider().getClass());
assertEquals(WALFactory.Providers.multiwal.clazz,
walFactory.getMetaProvider().getClass());
@@ -645,7 +645,7 @@ public class TestWALFactory {
public void testOnlySetMetaWALProvider() throws IOException {
Configuration conf = new Configuration();
conf.set(META_WAL_PROVIDER, WALFactory.Providers.asyncfs.name());
- WALFactory walFactory = new WALFactory(conf,
this.currentServername.toString());
+ WALFactory walFactory = new WALFactory(conf, this.currentServername, null);
assertEquals(WALFactory.Providers.defaultProvider.clazz,
walFactory.getWALProvider().getClass());
assertEquals(WALFactory.Providers.asyncfs.clazz,
walFactory.getMetaProvider().getClass());
@@ -655,19 +655,18 @@ public class TestWALFactory {
public void testDefaultProvider() throws IOException {
final Configuration conf = new Configuration();
// AsyncFSWal is the default, we should be able to request any WAL.
- final WALFactory normalWalFactory = new WALFactory(conf,
this.currentServername.toString());
+ final WALFactory normalWalFactory = new WALFactory(conf,
this.currentServername, null);
Class<? extends WALProvider> fshLogProvider =
normalWalFactory.getProviderClass(WALFactory.WAL_PROVIDER,
Providers.filesystem.name());
assertEquals(Providers.filesystem.clazz, fshLogProvider);
// Imagine a world where MultiWAL is the default
- final WALFactory customizedWalFactory =
- new WALFactory(conf, this.currentServername.toString()) {
- @Override
- Providers getDefaultProvider() {
- return Providers.multiwal;
- }
- };
+ final WALFactory customizedWalFactory = new WALFactory(conf,
this.currentServername, null) {
+ @Override
+ Providers getDefaultProvider() {
+ return Providers.multiwal;
+ }
+ };
// If we don't specify a WALProvider, we should get the default
implementation.
Class<? extends WALProvider> multiwalProviderClass =
customizedWalFactory.getProviderClass(WALFactory.WAL_PROVIDER,
Providers.multiwal.name());
@@ -678,7 +677,7 @@ public class TestWALFactory {
public void testCustomProvider() throws IOException {
final Configuration config = new Configuration();
config.set(WALFactory.WAL_PROVIDER, IOTestProvider.class.getName());
- final WALFactory walFactory = new WALFactory(config,
this.currentServername.toString());
+ final WALFactory walFactory = new WALFactory(config,
this.currentServername, null);
Class<? extends WALProvider> walProvider =
walFactory.getProviderClass(WALFactory.WAL_PROVIDER,
Providers.filesystem.name());
assertEquals(IOTestProvider.class, walProvider);
@@ -690,7 +689,7 @@ public class TestWALFactory {
public void testCustomMetaProvider() throws IOException {
final Configuration config = new Configuration();
config.set(WALFactory.META_WAL_PROVIDER, IOTestProvider.class.getName());
- final WALFactory walFactory = new WALFactory(config,
this.currentServername.toString());
+ final WALFactory walFactory = new WALFactory(config,
this.currentServername, null);
Class<? extends WALProvider> walProvider =
walFactory.getProviderClass(WALFactory.WAL_PROVIDER,
Providers.filesystem.name());
assertEquals(Providers.filesystem.clazz, walProvider);
@@ -702,7 +701,7 @@ public class TestWALFactory {
public void testCustomReplicationProvider() throws IOException {
final Configuration config = new Configuration();
config.set(WALFactory.REPLICATION_WAL_PROVIDER,
IOTestProvider.class.getName());
- final WALFactory walFactory = new WALFactory(config,
this.currentServername.toString());
+ final WALFactory walFactory = new WALFactory(config,
this.currentServername, null);
Class<? extends WALProvider> walProvider =
walFactory.getProviderClass(WALFactory.WAL_PROVIDER,
Providers.filesystem.name());
assertEquals(Providers.filesystem.clazz, walProvider);
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplitToHFile.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplitToHFile.java
index 3dca289cb45..cf9a6f174d6 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplitToHFile.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplitToHFile.java
@@ -173,12 +173,15 @@ public class TestWALSplitToHFile {
}
private WAL createWAL(Configuration c, Path hbaseRootDir, String logName)
throws IOException {
- FSHLog wal = new FSHLog(FileSystem.get(c), hbaseRootDir, logName, c);
+ FileSystem fs = hbaseRootDir.getFileSystem(c);
+ fs.mkdirs(new Path(hbaseRootDir, logName));
+ FSHLog wal = new FSHLog(fs, hbaseRootDir, logName, c);
wal.init();
return wal;
}
private WAL createWAL(FileSystem fs, Path hbaseRootDir, String logName)
throws IOException {
+ fs.mkdirs(new Path(hbaseRootDir, logName));
FSHLog wal = new FSHLog(fs, hbaseRootDir, logName, this.conf);
wal.init();
return wal;
diff --git a/pom.xml b/pom.xml
index c074108f057..3d4b0fdd823 100644
--- a/pom.xml
+++ b/pom.xml
@@ -845,6 +845,7 @@
<jruby.version>9.4.14.0</jruby.version>
<junit.jupiter.version>5.13.4</junit.jupiter.version>
<junit.vintage.version>5.13.4</junit.vintage.version>
+ <awaitility.version>4.3.0</awaitility.version>
<hamcrest.version>1.3</hamcrest.version>
<opentelemetry.version>1.49.0</opentelemetry.version>
<opentelemetry-semconv.version>1.29.0-alpha</opentelemetry-semconv.version>
@@ -1630,6 +1631,12 @@
<version>${junit.vintage.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.awaitility</groupId>
+ <artifactId>awaitility</artifactId>
+ <version>${awaitility.version}</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-core</artifactId>
