This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 8841debe872 HBASE-29802 NPE when shutting down mini cluster cause
tests hang (#7604) (#7612)
8841debe872 is described below
commit 8841debe872ccc3054e7a80dcfb471f28b3af34d
Author: Duo Zhang <[email protected]>
AuthorDate: Mon Jan 12 22:59:08 2026 +0800
HBASE-29802 NPE when shutting down mini cluster cause tests hang (#7604)
(#7612)
Remove the 'shutdownHook' in SingleProcessHBaseCluster where we close
the FileSystem instance of a region server.
Set DistributedFileSystem cache to false when creating MiniMRCluster so
when shutting down MiniMRCluster it will not close the shared FileSystem
instance.
Also re-enable TestExportSnapshot.
(cherry picked from commit d11ee6a4586bd2afd7201f525a68f76de98e279d)
Signed-off-by: Nick Dimiduk <[email protected]>
---
.../hadoop/hbase/snapshot/TestExportSnapshot.java | 2 -
.../apache/hadoop/hbase/HBaseTestingUtility.java | 6 ++-
.../org/apache/hadoop/hbase/MiniHBaseCluster.java | 47 +---------------------
3 files changed, 6 insertions(+), 49 deletions(-)
diff --git
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
index af1d460a58f..0cea6f6b8e9 100644
---
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
+++
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
@@ -64,7 +64,6 @@ import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -80,7 +79,6 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.Snapshot
/**
* Test Export Snapshot Tool
*/
-@Ignore // HBASE-24493
@Category({ VerySlowMapReduceTests.class, LargeTests.class })
public class TestExportSnapshot {
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 84aa1cbd821..a7e72f61048 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -2832,10 +2832,14 @@ public class HBaseTestingUtility extends
HBaseZKTestingUtility {
jvmOpts + " --add-opens java.base/java.lang=ALL-UNNAMED");
}
+ // Disable fs cache for DistributedFileSystem, to avoid YARN RM close the
shared FileSystem
+ // instance and cause trouble in HBase. See HBASE-29802 for more details.
+ JobConf mrClusterConf = new JobConf(conf);
+ mrClusterConf.setBoolean("fs.hdfs.impl.disable.cache", true);
// Allow the user to override FS URI for this map-reduce cluster to use.
mrCluster =
new MiniMRCluster(servers, FS_URI != null ? FS_URI :
FileSystem.get(conf).getUri().toString(),
- 1, null, null, new JobConf(this.conf));
+ 1, null, null, mrClusterConf);
JobConf jobConf = MapreduceTestingShim.getJobConf(mrCluster);
if (jobConf == null) {
jobConf = mrCluster.createJobConf();
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
index d6093d33118..38141624699 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
@@ -24,7 +24,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -45,7 +44,6 @@ import org.slf4j.LoggerFactory;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ClientService;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MasterService;
-import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerStartupResponse;
/**
* This class creates a single process HBase cluster. each server. The master
uses the 'default'
@@ -122,7 +120,7 @@ public class MiniHBaseCluster extends HBaseCluster {
* only, not All filesystems as the FileSystem system exit hook does.
*/
public static class MiniHBaseClusterRegionServer extends HRegionServer {
- private Thread shutdownThread = null;
+
private User user = null;
/**
* List of RegionServers killed so far. ServerName also comprises
startCode of a server, so any
@@ -137,20 +135,6 @@ public class MiniHBaseCluster extends HBaseCluster {
this.user = User.getCurrent();
}
- /*
- * @param currentfs We return this if we did not make a new one.
- * @param uniqueName Same name used to help identify the created fs.
- * @return A new fs instance if we are up on DistributeFileSystem.
- */
-
- @Override
- protected void handleReportForDutyResponse(final
RegionServerStartupResponse c)
- throws IOException {
- super.handleReportForDutyResponse(c);
- // Run this thread to shutdown our filesystem on way out.
- this.shutdownThread = new
SingleFileSystemShutdownThread(getFileSystem());
- }
-
@Override
public void run() {
try {
@@ -163,12 +147,6 @@ public class MiniHBaseCluster extends HBaseCluster {
});
} catch (Throwable t) {
LOG.error("Exception in run", t);
- } finally {
- // Run this on the way out.
- if (this.shutdownThread != null) {
- this.shutdownThread.start();
- Threads.shutdown(this.shutdownThread, 30000);
- }
}
}
@@ -198,29 +176,6 @@ public class MiniHBaseCluster extends HBaseCluster {
}
}
- /**
- * Alternate shutdown hook. Just shuts down the passed fs, not all as
default filesystem hook
- * does.
- */
- static class SingleFileSystemShutdownThread extends Thread {
- private final FileSystem fs;
-
- SingleFileSystemShutdownThread(final FileSystem fs) {
- super("Shutdown of " + fs);
- this.fs = fs;
- }
-
- @Override
- public void run() {
- try {
- LOG.info("Hook closing fs=" + this.fs);
- this.fs.close();
- } catch (IOException e) {
- LOG.warn("Running hook", e);
- }
- }
- }
-
private void init(final int nMasterNodes, final int numAlwaysStandByMasters,
final int nRegionNodes, List<Integer> rsPorts, Class<? extends HMaster>
masterClass,
Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer>
regionserverClass)
