(hbase) branch branch-2.6 updated: HBASE-20528 Revise collections copying from iteration to built-in function

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.6 by this push:
 new 604f66028c8 HBASE-20528 Revise collections copying from iteration to 
built-in function
604f66028c8 is described below

commit 604f66028c890195690fd57a12122c49c905dfc3
Author: Jacky Ho 
AuthorDate: Sat Dec 16 21:48:36 2023 +0800

HBASE-20528 Revise collections copying from iteration to built-in function

Signed-off-by: Duo Zhang 
(cherry picked from commit b7c3f8c6e33c34a1af8b88985b143ba71c4f3bad)
---
 .../src/main/java/org/apache/hadoop/hbase/client/Result.java | 4 +---
 .../main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java| 5 ++---
 .../java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java | 5 ++---
 .../src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java   | 5 ++---
 4 files changed, 7 insertions(+), 12 deletions(-)

diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
index bcaf7721c0a..6915adec018 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
@@ -791,9 +791,7 @@ public class Result implements CellScannable, CellScanner {
   }
   prevRow = currentRow;
   stale = stale || r.isStale();
-  for (Cell c : r.rawCells()) {
-cells.add(c);
-  }
+  Collections.addAll(cells, r.rawCells());
 }
 
 return Result.create(cells, null, stale);
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
index 7235de14803..650ec8120ca 100644
--- 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
+++ 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.util;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
@@ -120,9 +121,7 @@ public abstract class AbstractHBaseTool implements Tool {
 
 CommandLine cmd;
 List argsList = new ArrayList<>(args.length);
-for (String arg : args) {
-  argsList.add(arg);
-}
+Collections.addAll(argsList, args);
 // For backward compatibility of args which can't be parsed as Option. See 
javadoc for
 // processOldArgs(..)
 processOldArgs(argsList);
diff --git 
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java
 
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java
index 4ff4a5b95b9..3ccbaab4de1 100644
--- 
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java
+++ 
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.mapreduce;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
@@ -148,9 +149,7 @@ public class HFileInputFormat extends 
FileInputFormat {
 for (FileStatus status : super.listStatus(job)) {
   if (status.isDirectory()) {
 FileSystem fs = status.getPath().getFileSystem(job.getConfiguration());
-for (FileStatus match : fs.listStatus(status.getPath(), 
HIDDEN_FILE_FILTER)) {
-  result.add(match);
-}
+Collections.addAll(result, fs.listStatus(status.getPath(), 
HIDDEN_FILE_FILTER));
   } else {
 result.add(status);
   }
diff --git 
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
 
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
index 5f2848c22e1..696e5257244 100644
--- 
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
+++ 
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -172,9 +173,7 @@ public class WALPlayer extends Configured implements Tool {
   Configuration conf = context.getConfiguration();
   String[] tables = conf.getStrings(TABLES_KEY);
   this.multiTableSupport = conf.getBoolean(MULTI_TABLES_SUPPORT, false);
-  for (String table : tables) {
-tableSet.add(table);
-  }
+  Collections.addAll(tableSet, tables);
 }
   }
 

(hbase) branch branch-2.5 updated: HBASE-20528 Revise collections copying from iteration to built-in function

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.5 by this push:
 new 6f84f6483ba HBASE-20528 Revise collections copying from iteration to 
built-in function
6f84f6483ba is described below

commit 6f84f6483bafdde5cd63bae8f27838f21b068c62
Author: Jacky Ho 
AuthorDate: Sat Dec 16 21:48:36 2023 +0800

HBASE-20528 Revise collections copying from iteration to built-in function

Signed-off-by: Duo Zhang 
(cherry picked from commit b7c3f8c6e33c34a1af8b88985b143ba71c4f3bad)
---
 .../src/main/java/org/apache/hadoop/hbase/client/Result.java | 4 +---
 .../main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java| 5 ++---
 .../java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java | 5 ++---
 3 files changed, 5 insertions(+), 9 deletions(-)

diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
index d2f4f85548d..48db34d8b59 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
@@ -791,9 +791,7 @@ public class Result implements CellScannable, CellScanner {
   }
   prevRow = currentRow;
   stale = stale || r.isStale();
-  for (Cell c : r.rawCells()) {
-cells.add(c);
-  }
+  Collections.addAll(cells, r.rawCells());
 }
 
 return Result.create(cells, null, stale);
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
index 7235de14803..650ec8120ca 100644
--- 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
+++ 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.util;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
@@ -120,9 +121,7 @@ public abstract class AbstractHBaseTool implements Tool {
 
 CommandLine cmd;
 List argsList = new ArrayList<>(args.length);
-for (String arg : args) {
-  argsList.add(arg);
-}
+Collections.addAll(argsList, args);
 // For backward compatibility of args which can't be parsed as Option. See 
javadoc for
 // processOldArgs(..)
 processOldArgs(argsList);
diff --git 
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java
 
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java
index 4ff4a5b95b9..3ccbaab4de1 100644
--- 
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java
+++ 
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.mapreduce;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
@@ -148,9 +149,7 @@ public class HFileInputFormat extends 
FileInputFormat {
 for (FileStatus status : super.listStatus(job)) {
   if (status.isDirectory()) {
 FileSystem fs = status.getPath().getFileSystem(job.getConfiguration());
-for (FileStatus match : fs.listStatus(status.getPath(), 
HIDDEN_FILE_FILTER)) {
-  result.add(match);
-}
+Collections.addAll(result, fs.listStatus(status.getPath(), 
HIDDEN_FILE_FILTER));
   } else {
 result.add(status);
   }

(hbase) branch branch-2.4 updated: HBASE-20528 Revise collections copying from iteration to built-in function

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.4 by this push:
 new 93a1b976591 HBASE-20528 Revise collections copying from iteration to 
built-in function
93a1b976591 is described below

commit 93a1b976591aff8b8ac96df5cf2cbc78180e3e7f
Author: Jacky Ho 
AuthorDate: Sat Dec 16 21:48:36 2023 +0800

HBASE-20528 Revise collections copying from iteration to built-in function

Signed-off-by: Duo Zhang 
(cherry picked from commit b7c3f8c6e33c34a1af8b88985b143ba71c4f3bad)
---
 .../src/main/java/org/apache/hadoop/hbase/client/Result.java | 4 +---
 .../main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java| 5 ++---
 .../java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java | 5 ++---
 3 files changed, 5 insertions(+), 9 deletions(-)

diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
index 51821fc7429..f43fa3efbfe 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
@@ -791,9 +791,7 @@ public class Result implements CellScannable, CellScanner {
   }
   prevRow = currentRow;
   stale = stale || r.isStale();
-  for (Cell c : r.rawCells()) {
-cells.add(c);
-  }
+  Collections.addAll(cells, r.rawCells());
 }
 
 return Result.create(cells, null, stale);
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
index 7235de14803..650ec8120ca 100644
--- 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
+++ 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.util;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
@@ -120,9 +121,7 @@ public abstract class AbstractHBaseTool implements Tool {
 
 CommandLine cmd;
 List argsList = new ArrayList<>(args.length);
-for (String arg : args) {
-  argsList.add(arg);
-}
+Collections.addAll(argsList, args);
 // For backward compatibility of args which can't be parsed as Option. See 
javadoc for
 // processOldArgs(..)
 processOldArgs(argsList);
diff --git 
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java
 
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java
index 44152727fdf..cad4532eff4 100644
--- 
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java
+++ 
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.mapreduce;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
@@ -148,9 +149,7 @@ public class HFileInputFormat extends 
FileInputFormat {
 for (FileStatus status : super.listStatus(job)) {
   if (status.isDirectory()) {
 FileSystem fs = status.getPath().getFileSystem(job.getConfiguration());
-for (FileStatus match : fs.listStatus(status.getPath(), 
HIDDEN_FILE_FILTER)) {
-  result.add(match);
-}
+Collections.addAll(result, fs.listStatus(status.getPath(), 
HIDDEN_FILE_FILTER));
   } else {
 result.add(status);
   }

(hbase) branch master updated: HBASE-21243 Correct java-doc for the method RpcServer.getRemoteAddress()

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
 new d747e15251e HBASE-21243 Correct java-doc for the method 
RpcServer.getRemoteAddress()
d747e15251e is described below

commit d747e15251e0bda9c76c052b2bd6dac1fac5b408
Author: Nihal Jain 
AuthorDate: Sat Dec 16 20:14:19 2023 +0800

HBASE-21243 Correct java-doc for the method RpcServer.getRemoteAddress()

Signed-off-by: Duo Zhang 
---
 .../src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
index f48183d0ee6..6b4bf28bc95 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
@@ -790,7 +790,10 @@ public abstract class RpcServer implements 
RpcServerInterface, ConfigurationObse
 return getRequestUser().map(User::getShortName);
   }
 
-  /** Returns Address of remote client if a request is ongoing, else null */
+  /**
+   * Returns the address of the remote client associated with the current RPC 
request or not present
+   * if no address is set.
+   */
   public static Optional getRemoteAddress() {
 return getCurrentCall().map(RpcCall::getRemoteAddress);
   }

(hbase) branch branch-3 updated: HBASE-21243 Correct java-doc for the method RpcServer.getRemoteAddress()

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-3 by this push:
 new 57fa5cff7f6 HBASE-21243 Correct java-doc for the method 
RpcServer.getRemoteAddress()
57fa5cff7f6 is described below

commit 57fa5cff7f649b8a036e478cfe1a3aaffd40e515
Author: Nihal Jain 
AuthorDate: Sat Dec 16 20:14:19 2023 +0800

HBASE-21243 Correct java-doc for the method RpcServer.getRemoteAddress()

Signed-off-by: Duo Zhang 
(cherry picked from commit d747e15251e0bda9c76c052b2bd6dac1fac5b408)
---
 .../src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
index f48183d0ee6..6b4bf28bc95 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
@@ -790,7 +790,10 @@ public abstract class RpcServer implements 
RpcServerInterface, ConfigurationObse
 return getRequestUser().map(User::getShortName);
   }
 
-  /** Returns Address of remote client if a request is ongoing, else null */
+  /**
+   * Returns the address of the remote client associated with the current RPC 
request or not present
+   * if no address is set.
+   */
   public static Optional getRemoteAddress() {
 return getCurrentCall().map(RpcCall::getRemoteAddress);
   }

(hbase) branch branch-2 updated: HBASE-21243 Correct java-doc for the method RpcServer.getRemoteAddress()

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2 by this push:
 new d984949c373 HBASE-21243 Correct java-doc for the method 
RpcServer.getRemoteAddress()
d984949c373 is described below

commit d984949c373260fdd663131d4f0cd5a19debeff6
Author: Nihal Jain 
AuthorDate: Sat Dec 16 20:14:19 2023 +0800

HBASE-21243 Correct java-doc for the method RpcServer.getRemoteAddress()

Signed-off-by: Duo Zhang 
(cherry picked from commit d747e15251e0bda9c76c052b2bd6dac1fac5b408)
---
 .../src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
index 54c18edc12c..94b197f1e3d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
@@ -779,7 +779,10 @@ public abstract class RpcServer implements 
RpcServerInterface, ConfigurationObse
 return getRequestUser().map(User::getShortName);
   }
 
-  /** Returns Address of remote client if a request is ongoing, else null */
+  /**
+   * Returns the address of the remote client associated with the current RPC 
request or not present
+   * if no address is set.
+   */
   public static Optional getRemoteAddress() {
 return getCurrentCall().map(RpcCall::getRemoteAddress);
   }

(hbase) branch branch-2.6 updated: HBASE-21243 Correct java-doc for the method RpcServer.getRemoteAddress()

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.6 by this push:
 new 3958f911621 HBASE-21243 Correct java-doc for the method 
RpcServer.getRemoteAddress()
3958f911621 is described below

commit 3958f911621416ffbd78e87f88933da0efda5b74
Author: Nihal Jain 
AuthorDate: Sat Dec 16 20:14:19 2023 +0800

HBASE-21243 Correct java-doc for the method RpcServer.getRemoteAddress()

Signed-off-by: Duo Zhang 
(cherry picked from commit d747e15251e0bda9c76c052b2bd6dac1fac5b408)
---
 .../src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
index 54c18edc12c..94b197f1e3d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
@@ -779,7 +779,10 @@ public abstract class RpcServer implements 
RpcServerInterface, ConfigurationObse
 return getRequestUser().map(User::getShortName);
   }
 
-  /** Returns Address of remote client if a request is ongoing, else null */
+  /**
+   * Returns the address of the remote client associated with the current RPC 
request or not present
+   * if no address is set.
+   */
   public static Optional getRemoteAddress() {
 return getCurrentCall().map(RpcCall::getRemoteAddress);
   }

(hbase) branch branch-2.5 updated: HBASE-21243 Correct java-doc for the method RpcServer.getRemoteAddress()

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.5 by this push:
 new e86f25746ad HBASE-21243 Correct java-doc for the method 
RpcServer.getRemoteAddress()
e86f25746ad is described below

commit e86f25746ad27dbb1c9a38a82bfdfedfe5e54742
Author: Nihal Jain 
AuthorDate: Sat Dec 16 20:14:19 2023 +0800

HBASE-21243 Correct java-doc for the method RpcServer.getRemoteAddress()

Signed-off-by: Duo Zhang 
(cherry picked from commit d747e15251e0bda9c76c052b2bd6dac1fac5b408)
---
 .../src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
index 39aca2c54fb..e437f260bbd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
@@ -739,7 +739,10 @@ public abstract class RpcServer implements 
RpcServerInterface, ConfigurationObse
 return getRequestUser().map(User::getShortName);
   }
 
-  /** Returns Address of remote client if a request is ongoing, else null */
+  /**
+   * Returns the address of the remote client associated with the current RPC 
request or not present
+   * if no address is set.
+   */
   public static Optional getRemoteAddress() {
 return getCurrentCall().map(RpcCall::getRemoteAddress);
   }

(hbase) branch branch-2.4 updated: HBASE-21243 Correct java-doc for the method RpcServer.getRemoteAddress()

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.4 by this push:
 new 79de88ec9b6 HBASE-21243 Correct java-doc for the method 
RpcServer.getRemoteAddress()
79de88ec9b6 is described below

commit 79de88ec9b62a9d30e9b03e1aab9d11f35c6c17b
Author: Nihal Jain 
AuthorDate: Sat Dec 16 20:14:19 2023 +0800

HBASE-21243 Correct java-doc for the method RpcServer.getRemoteAddress()

Signed-off-by: Duo Zhang 
(cherry picked from commit d747e15251e0bda9c76c052b2bd6dac1fac5b408)
---
 .../src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
index daa71d5983c..175e9d357f3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
@@ -711,7 +711,10 @@ public abstract class RpcServer implements 
RpcServerInterface, ConfigurationObse
 return getRequestUser().map(User::getShortName);
   }
 
-  /** Returns Address of remote client if a request is ongoing, else null */
+  /**
+   * Returns the address of the remote client associated with the current RPC 
request or not present
+   * if no address is set.
+   */
   public static Optional getRemoteAddress() {
 return getCurrentCall().map(RpcCall::getRemoteAddress);
   }

(hbase) branch branch-3 updated: HBASE-28180 Review the usage of RegionStates.getOrCreateServer (#5486)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-3 by this push:
 new 9a05aef2c96 HBASE-28180 Review the usage of 
RegionStates.getOrCreateServer (#5486)
9a05aef2c96 is described below

commit 9a05aef2c96a928040b93bd9b1f7e0f35f776a11
Author: Duo Zhang 
AuthorDate: Sat Dec 16 16:28:50 2023 +0800

HBASE-28180 Review the usage of RegionStates.getOrCreateServer (#5486)

Signed-off-by: Xiaolin Ha 
(cherry picked from commit e45b9c42d517fc131e60bf76383e55975ded36b1)
---
 .../org/apache/hadoop/hbase/master/HMaster.java|  12 ++
 .../hadoop/hbase/master/RegionServerTracker.java   |   2 +-
 .../apache/hadoop/hbase/master/ServerManager.java  |   5 +
 .../hbase/master/assignment/AssignmentManager.java | 155 +
 .../hbase/master/assignment/RegionStates.java  |  33 +++--
 .../hbase/master/assignment/ServerStateNode.java   |   1 +
 .../master/procedure/HBCKServerCrashProcedure.java |   2 +
 .../apache/hadoop/hbase/master/TestBalancer.java   |   2 +-
 .../hbase/master/TestClockSkewDetection.java   |  23 ++-
 .../hbase/master/TestClusterRestartFailover.java   |  37 +++--
 .../procedure/MasterProcedureTestingUtility.java   |   5 +
 .../procedure/TestServerRemoteProcedure.java   |   2 +-
 12 files changed, 191 insertions(+), 88 deletions(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index b492b177e42..8567f00cad0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -3342,6 +3342,18 @@ public class HMaster extends 
HBaseServerBase implements Maste
 procedureExecutor.getEnvironment().setEventReady(initialized, 
isInitialized);
   }
 
+  /**
+   * Mainly used in procedure related tests, where we will restart 
ProcedureExecutor and
+   * AssignmentManager, but we do not want to restart master(to speed up the 
test), so we need to
+   * disable rpc for a while otherwise some critical rpc requests such as
+   * reportRegionStateTransition could fail and cause region server to abort.
+   */
+  @RestrictedApi(explanation = "Should only be called in tests", link = "",
+  allowedOnPath = ".*/src/test/.*")
+  public void setServiceStarted(boolean started) {
+this.serviceStarted = started;
+  }
+
   @Override
   public ProcedureEvent getInitializedEvent() {
 return initialized;
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java
index 71ca500a045..a09f6689a5c 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java
@@ -135,7 +135,7 @@ public class RegionServerTracker extends ZKListener {
   .forEach(s -> LOG.error("{} has no matching ServerCrashProcedure", s));
 // create ServerNode for all possible live servers from wal directory and 
master local region
 liveServersBeforeRestart
-  .forEach(sn -> 
server.getAssignmentManager().getRegionStates().getOrCreateServer(sn));
+  .forEach(sn -> 
server.getAssignmentManager().getRegionStates().createServer(sn));
 ServerManager serverManager = server.getServerManager();
 synchronized (this) {
   Set liveServers = regionServers;
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 6a169beb53b..2afd48c58df 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -426,6 +426,7 @@ public class ServerManager {
   void recordNewServerWithLock(final ServerName serverName, final 
ServerMetrics sl) {
 LOG.info("Registering regionserver=" + serverName);
 this.onlineServers.put(serverName, sl);
+master.getAssignmentManager().getRegionStates().createServer(serverName);
   }
 
   public ConcurrentNavigableMap getFlushedSequenceIdByRegion() {
@@ -603,6 +604,10 @@ public class ServerManager {
 }
 LOG.info("Processing expiration of " + serverName + " on " + 
this.master.getServerName());
 long pid = master.getAssignmentManager().submitServerCrash(serverName, 
true, force);
+if (pid == Procedure.NO_PROC_ID) {
+  // skip later processing as we failed to submit SCP
+  return Procedure.NO_PROC_ID;
+}
 storage.expired(serverName);
 // Tell our listeners that a server was removed
 if (!

(hbase) branch branch-2.6 updated: HBASE-28180 Review the usage of RegionStates.getOrCreateServer (#5486)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.6 by this push:
 new db63ebefaf6 HBASE-28180 Review the usage of 
RegionStates.getOrCreateServer (#5486)
db63ebefaf6 is described below

commit db63ebefaf6c586972604d6ac06d6bb490078063
Author: Duo Zhang 
AuthorDate: Sat Dec 16 16:28:50 2023 +0800

HBASE-28180 Review the usage of RegionStates.getOrCreateServer (#5486)

Signed-off-by: Xiaolin Ha 
(cherry picked from commit e45b9c42d517fc131e60bf76383e55975ded36b1)
---
 .../org/apache/hadoop/hbase/master/HMaster.java|  12 ++
 .../hadoop/hbase/master/RegionServerTracker.java   |   2 +-
 .../apache/hadoop/hbase/master/ServerManager.java  |   5 +
 .../hbase/master/assignment/AssignmentManager.java | 154 +
 .../hbase/master/assignment/RegionStates.java  |  33 +++--
 .../hbase/master/assignment/ServerStateNode.java   |   1 +
 .../master/procedure/HBCKServerCrashProcedure.java |   2 +
 .../apache/hadoop/hbase/master/TestBalancer.java   |   2 +-
 .../hbase/master/TestClockSkewDetection.java   |  31 +++--
 .../hbase/master/TestClusterRestartFailover.java   |  37 +++--
 .../procedure/MasterProcedureTestingUtility.java   |   5 +
 .../procedure/TestServerRemoteProcedure.java   |   2 +-
 12 files changed, 189 insertions(+), 97 deletions(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 15038da705d..063c423c244 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -3282,6 +3282,18 @@ public class HMaster extends HRegionServer implements 
MasterServices {
 procedureExecutor.getEnvironment().setEventReady(initialized, 
isInitialized);
   }
 
+  /**
+   * Mainly used in procedure related tests, where we will restart 
ProcedureExecutor and
+   * AssignmentManager, but we do not want to restart master(to speed up the 
test), so we need to
+   * disable rpc for a while otherwise some critical rpc requests such as
+   * reportRegionStateTransition could fail and cause region server to abort.
+   */
+  @RestrictedApi(explanation = "Should only be called in tests", link = "",
+  allowedOnPath = ".*/src/test/.*")
+  public void setServiceStarted(boolean started) {
+this.serviceStarted = started;
+  }
+
   @Override
   public ProcedureEvent getInitializedEvent() {
 return initialized;
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java
index 5ecf6a2f6e6..c1d997797c8 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java
@@ -135,7 +135,7 @@ public class RegionServerTracker extends ZKListener {
   .forEach(s -> LOG.error("{} has no matching ServerCrashProcedure", s));
 // create ServerNode for all possible live servers from wal directory
 liveServersBeforeRestart
-  .forEach(sn -> 
server.getAssignmentManager().getRegionStates().getOrCreateServer(sn));
+  .forEach(sn -> 
server.getAssignmentManager().getRegionStates().createServer(sn));
 ServerManager serverManager = server.getServerManager();
 synchronized (this) {
   Set liveServers = regionServers;
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 196a1a582ed..c7534c11acc 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -395,6 +395,7 @@ public class ServerManager {
   void recordNewServerWithLock(final ServerName serverName, final 
ServerMetrics sl) {
 LOG.info("Registering regionserver=" + serverName);
 this.onlineServers.put(serverName, sl);
+master.getAssignmentManager().getRegionStates().createServer(serverName);
   }
 
   public RegionStoreSequenceIds getLastFlushedSequenceId(byte[] 
encodedRegionName) {
@@ -567,6 +568,10 @@ public class ServerManager {
 }
 LOG.info("Processing expiration of " + serverName + " on " + 
this.master.getServerName());
 long pid = master.getAssignmentManager().submitServerCrash(serverName, 
true, force);
+if (pid == Procedure.NO_PROC_ID) {
+  // skip later processing as we failed to submit SCP
+  return Procedure.NO_PROC_ID;
+}
 storage.expired(serverName);
 // Tell our listeners that a server was removed

(hbase) branch branch-2 updated: HBASE-28180 Review the usage of RegionStates.getOrCreateServer (#5486)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2 by this push:
 new e836a057a71 HBASE-28180 Review the usage of 
RegionStates.getOrCreateServer (#5486)
e836a057a71 is described below

commit e836a057a713be2290097e5f0aa8e711f36d092d
Author: Duo Zhang 
AuthorDate: Sat Dec 16 16:28:50 2023 +0800

HBASE-28180 Review the usage of RegionStates.getOrCreateServer (#5486)

Signed-off-by: Xiaolin Ha 
(cherry picked from commit e45b9c42d517fc131e60bf76383e55975ded36b1)
---
 .../org/apache/hadoop/hbase/master/HMaster.java|  12 ++
 .../hadoop/hbase/master/RegionServerTracker.java   |   2 +-
 .../apache/hadoop/hbase/master/ServerManager.java  |   5 +
 .../hbase/master/assignment/AssignmentManager.java | 154 +
 .../hbase/master/assignment/RegionStates.java  |  33 +++--
 .../hbase/master/assignment/ServerStateNode.java   |   1 +
 .../master/procedure/HBCKServerCrashProcedure.java |   2 +
 .../apache/hadoop/hbase/master/TestBalancer.java   |   2 +-
 .../hbase/master/TestClockSkewDetection.java   |  31 +++--
 .../hbase/master/TestClusterRestartFailover.java   |  37 +++--
 .../procedure/MasterProcedureTestingUtility.java   |   5 +
 .../procedure/TestServerRemoteProcedure.java   |   2 +-
 12 files changed, 189 insertions(+), 97 deletions(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 15038da705d..063c423c244 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -3282,6 +3282,18 @@ public class HMaster extends HRegionServer implements 
MasterServices {
 procedureExecutor.getEnvironment().setEventReady(initialized, 
isInitialized);
   }
 
+  /**
+   * Mainly used in procedure related tests, where we will restart 
ProcedureExecutor and
+   * AssignmentManager, but we do not want to restart master(to speed up the 
test), so we need to
+   * disable rpc for a while otherwise some critical rpc requests such as
+   * reportRegionStateTransition could fail and cause region server to abort.
+   */
+  @RestrictedApi(explanation = "Should only be called in tests", link = "",
+  allowedOnPath = ".*/src/test/.*")
+  public void setServiceStarted(boolean started) {
+this.serviceStarted = started;
+  }
+
   @Override
   public ProcedureEvent getInitializedEvent() {
 return initialized;
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java
index 5ecf6a2f6e6..c1d997797c8 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java
@@ -135,7 +135,7 @@ public class RegionServerTracker extends ZKListener {
   .forEach(s -> LOG.error("{} has no matching ServerCrashProcedure", s));
 // create ServerNode for all possible live servers from wal directory
 liveServersBeforeRestart
-  .forEach(sn -> 
server.getAssignmentManager().getRegionStates().getOrCreateServer(sn));
+  .forEach(sn -> 
server.getAssignmentManager().getRegionStates().createServer(sn));
 ServerManager serverManager = server.getServerManager();
 synchronized (this) {
   Set liveServers = regionServers;
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 196a1a582ed..c7534c11acc 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -395,6 +395,7 @@ public class ServerManager {
   void recordNewServerWithLock(final ServerName serverName, final 
ServerMetrics sl) {
 LOG.info("Registering regionserver=" + serverName);
 this.onlineServers.put(serverName, sl);
+master.getAssignmentManager().getRegionStates().createServer(serverName);
   }
 
   public RegionStoreSequenceIds getLastFlushedSequenceId(byte[] 
encodedRegionName) {
@@ -567,6 +568,10 @@ public class ServerManager {
 }
 LOG.info("Processing expiration of " + serverName + " on " + 
this.master.getServerName());
 long pid = master.getAssignmentManager().submitServerCrash(serverName, 
true, force);
+if (pid == Procedure.NO_PROC_ID) {
+  // skip later processing as we failed to submit SCP
+  return Procedure.NO_PROC_ID;
+}
 storage.expired(serverName);
 // Tell our listeners that a server was removed
   

(hbase) branch master updated: HBASE-28180 Review the usage of RegionStates.getOrCreateServer (#5486)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
 new e45b9c42d51 HBASE-28180 Review the usage of 
RegionStates.getOrCreateServer (#5486)
e45b9c42d51 is described below

commit e45b9c42d517fc131e60bf76383e55975ded36b1
Author: Duo Zhang 
AuthorDate: Sat Dec 16 16:28:50 2023 +0800

HBASE-28180 Review the usage of RegionStates.getOrCreateServer (#5486)

Signed-off-by: Xiaolin Ha 
---
 .../org/apache/hadoop/hbase/master/HMaster.java|  12 ++
 .../hadoop/hbase/master/RegionServerTracker.java   |   2 +-
 .../apache/hadoop/hbase/master/ServerManager.java  |   5 +
 .../hbase/master/assignment/AssignmentManager.java | 155 +
 .../hbase/master/assignment/RegionStates.java  |  33 +++--
 .../hbase/master/assignment/ServerStateNode.java   |   1 +
 .../master/procedure/HBCKServerCrashProcedure.java |   2 +
 .../apache/hadoop/hbase/master/TestBalancer.java   |   2 +-
 .../hbase/master/TestClockSkewDetection.java   |  23 ++-
 .../hbase/master/TestClusterRestartFailover.java   |  37 +++--
 .../procedure/MasterProcedureTestingUtility.java   |   5 +
 .../procedure/TestServerRemoteProcedure.java   |   2 +-
 12 files changed, 191 insertions(+), 88 deletions(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index b492b177e42..8567f00cad0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -3342,6 +3342,18 @@ public class HMaster extends 
HBaseServerBase implements Maste
 procedureExecutor.getEnvironment().setEventReady(initialized, 
isInitialized);
   }
 
+  /**
+   * Mainly used in procedure related tests, where we will restart 
ProcedureExecutor and
+   * AssignmentManager, but we do not want to restart master(to speed up the 
test), so we need to
+   * disable rpc for a while otherwise some critical rpc requests such as
+   * reportRegionStateTransition could fail and cause region server to abort.
+   */
+  @RestrictedApi(explanation = "Should only be called in tests", link = "",
+  allowedOnPath = ".*/src/test/.*")
+  public void setServiceStarted(boolean started) {
+this.serviceStarted = started;
+  }
+
   @Override
   public ProcedureEvent getInitializedEvent() {
 return initialized;
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java
index 71ca500a045..a09f6689a5c 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java
@@ -135,7 +135,7 @@ public class RegionServerTracker extends ZKListener {
   .forEach(s -> LOG.error("{} has no matching ServerCrashProcedure", s));
 // create ServerNode for all possible live servers from wal directory and 
master local region
 liveServersBeforeRestart
-  .forEach(sn -> 
server.getAssignmentManager().getRegionStates().getOrCreateServer(sn));
+  .forEach(sn -> 
server.getAssignmentManager().getRegionStates().createServer(sn));
 ServerManager serverManager = server.getServerManager();
 synchronized (this) {
   Set liveServers = regionServers;
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 6a169beb53b..2afd48c58df 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -426,6 +426,7 @@ public class ServerManager {
   void recordNewServerWithLock(final ServerName serverName, final 
ServerMetrics sl) {
 LOG.info("Registering regionserver=" + serverName);
 this.onlineServers.put(serverName, sl);
+master.getAssignmentManager().getRegionStates().createServer(serverName);
   }
 
   public ConcurrentNavigableMap getFlushedSequenceIdByRegion() {
@@ -603,6 +604,10 @@ public class ServerManager {
 }
 LOG.info("Processing expiration of " + serverName + " on " + 
this.master.getServerName());
 long pid = master.getAssignmentManager().submitServerCrash(serverName, 
true, force);
+if (pid == Procedure.NO_PROC_ID) {
+  // skip later processing as we failed to submit SCP
+  return Procedure.NO_PROC_ID;
+}
 storage.expired(serverName);
 // Tell our listeners that a server was removed
 if (!this.listeners.isEmpty()) {
diff --git 
a/hbase-server/src/main/java/org/ap

(hbase) branch branch-2.5 updated: HBASE-28031 TestClusterScopeQuotaThrottle is still failing with broken WAL writer (#5529)(#5539)(#5582)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.5 by this push:
 new 4994220386a HBASE-28031 TestClusterScopeQuotaThrottle is still failing 
with broken WAL writer (#5529)(#5539)(#5582)
4994220386a is described below

commit 4994220386aa7a0f60cce894ad3fbfc6c8769711
Author: Duo Zhang 
AuthorDate: Mon Nov 20 10:41:35 2023 +0800

HBASE-28031 TestClusterScopeQuotaThrottle is still failing with broken WAL 
writer (#5529)(#5539)(#5582)

Limit the scope for EnvironmentEdge injection
Add timeout for ThrottleQuotaTestUtil.triggerCacheRefresh

Signed-off-by: Guanghao Zhang 
Signed-off-by: Bryan Beaudreault 
(cherry picked from commit b1eccb364b30b14bdf475c339dbea12daa6804e2)
---
 .../util/EnvironmentEdgeManagerTestHelper.java |  32 ++
 .../hadoop/hbase/quotas/ThrottleQuotaTestUtil.java | 107 +
 2 files changed, 101 insertions(+), 38 deletions(-)

diff --git 
a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java
 
b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java
index 684247248dc..73e7f1623ef 100644
--- 
a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java
+++ 
b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java
@@ -33,4 +33,36 @@ public final class EnvironmentEdgeManagerTestHelper {
   public static void injectEdge(EnvironmentEdge edge) {
 EnvironmentEdgeManager.injectEdge(edge);
   }
+
+  private static final class PackageEnvironmentEdgeWrapper implements 
EnvironmentEdge {
+
+private final EnvironmentEdge delegate;
+
+private final String packageName;
+
+PackageEnvironmentEdgeWrapper(EnvironmentEdge delegate, String 
packageName) {
+  this.delegate = delegate;
+  this.packageName = packageName;
+}
+
+@Override
+public long currentTime() {
+  StackTraceElement[] elements = new Exception().getStackTrace();
+  // the first element is us, the second one is EnvironmentEdgeManager, so 
let's check the third
+  // one
+  if (elements.length > 2 && 
elements[2].getClassName().startsWith(packageName)) {
+return delegate.currentTime();
+  } else {
+return System.currentTimeMillis();
+  }
+}
+  }
+
+  /**
+   * Inject a {@link EnvironmentEdge} which only takes effect when calling 
directly from the classes
+   * in the given package.
+   */
+  public static void injectEdgeForPackage(EnvironmentEdge edge, String 
packageName) {
+injectEdge(new PackageEnvironmentEdgeWrapper(edge, packageName));
+  }
 }
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
index d349fc25aa1..baf11df848c 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.util.Objects;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Table;
@@ -42,7 +43,9 @@ public final class ThrottleQuotaTestUtil {
   private final static int REFRESH_TIME = 30 * 6;
   static {
 envEdge.setValue(EnvironmentEdgeManager.currentTime());
-EnvironmentEdgeManagerTestHelper.injectEdge(envEdge);
+// only active the envEdge for quotas package
+EnvironmentEdgeManagerTestHelper.injectEdgeForPackage(envEdge,
+  ThrottleQuotaTestUtil.class.getPackage().getName());
   }
 
   private ThrottleQuotaTestUtil() {
@@ -135,51 +138,79 @@ public final class ThrottleQuotaTestUtil {
   RegionServerRpcQuotaManager quotaManager =
 rst.getRegionServer().getRegionServerRpcQuotaManager();
   QuotaCache quotaCache = quotaManager.getQuotaCache();
-
   quotaCache.triggerCacheRefresh();
-  // sleep for cache update
   Thread.sleep(250);
-
-  for (TableName table : tables) {
-quotaCache.getTableLimiter(table);
-  }
-
-  boolean isUpdated = false;
-  while (!isUpdated) {
-quotaCache.triggerCacheRefresh();
-isUpdated = true;
-for (TableName table : tables) {
-  boolean isBypass = true;
-  if (userLimiter) {
-isBypass = quotaCache.getUserLimiter(User.getCurrent().getUGI(), 
table).isBypass();
+  testUtil.waitFor(6, 250, new ExplainingPredicate() {
+
+@Override
+public boolean evaluate() throw

(hbase) branch branch-2 updated: HBASE-28031 TestClusterScopeQuotaThrottle is still failing with broken WAL writer (#5529)(#5539)(#5582)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2 by this push:
 new 67c09f8f866 HBASE-28031 TestClusterScopeQuotaThrottle is still failing 
with broken WAL writer (#5529)(#5539)(#5582)
67c09f8f866 is described below

commit 67c09f8f866272f8a124f79a503b23f191c4ba1a
Author: Duo Zhang 
AuthorDate: Mon Nov 20 10:41:35 2023 +0800

HBASE-28031 TestClusterScopeQuotaThrottle is still failing with broken WAL 
writer (#5529)(#5539)(#5582)

Limit the scope for EnvironmentEdge injection
Add timeout for ThrottleQuotaTestUtil.triggerCacheRefresh

Signed-off-by: Guanghao Zhang 
Signed-off-by: Bryan Beaudreault 
(cherry picked from commit b1eccb364b30b14bdf475c339dbea12daa6804e2)
---
 .../util/EnvironmentEdgeManagerTestHelper.java |  32 ++
 .../hadoop/hbase/quotas/ThrottleQuotaTestUtil.java | 107 +
 2 files changed, 101 insertions(+), 38 deletions(-)

diff --git 
a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java
 
b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java
index 684247248dc..73e7f1623ef 100644
--- 
a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java
+++ 
b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java
@@ -33,4 +33,36 @@ public final class EnvironmentEdgeManagerTestHelper {
   public static void injectEdge(EnvironmentEdge edge) {
 EnvironmentEdgeManager.injectEdge(edge);
   }
+
+  private static final class PackageEnvironmentEdgeWrapper implements 
EnvironmentEdge {
+
+private final EnvironmentEdge delegate;
+
+private final String packageName;
+
+PackageEnvironmentEdgeWrapper(EnvironmentEdge delegate, String 
packageName) {
+  this.delegate = delegate;
+  this.packageName = packageName;
+}
+
+@Override
+public long currentTime() {
+  StackTraceElement[] elements = new Exception().getStackTrace();
+  // the first element is us, the second one is EnvironmentEdgeManager, so 
let's check the third
+  // one
+  if (elements.length > 2 && 
elements[2].getClassName().startsWith(packageName)) {
+return delegate.currentTime();
+  } else {
+return System.currentTimeMillis();
+  }
+}
+  }
+
+  /**
+   * Inject a {@link EnvironmentEdge} which only takes effect when calling 
directly from the classes
+   * in the given package.
+   */
+  public static void injectEdgeForPackage(EnvironmentEdge edge, String 
packageName) {
+injectEdge(new PackageEnvironmentEdgeWrapper(edge, packageName));
+  }
 }
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
index d349fc25aa1..baf11df848c 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.util.Objects;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Table;
@@ -42,7 +43,9 @@ public final class ThrottleQuotaTestUtil {
   private final static int REFRESH_TIME = 30 * 6;
   static {
 envEdge.setValue(EnvironmentEdgeManager.currentTime());
-EnvironmentEdgeManagerTestHelper.injectEdge(envEdge);
+// only active the envEdge for quotas package
+EnvironmentEdgeManagerTestHelper.injectEdgeForPackage(envEdge,
+  ThrottleQuotaTestUtil.class.getPackage().getName());
   }
 
   private ThrottleQuotaTestUtil() {
@@ -135,51 +138,79 @@ public final class ThrottleQuotaTestUtil {
   RegionServerRpcQuotaManager quotaManager =
 rst.getRegionServer().getRegionServerRpcQuotaManager();
   QuotaCache quotaCache = quotaManager.getQuotaCache();
-
   quotaCache.triggerCacheRefresh();
-  // sleep for cache update
   Thread.sleep(250);
-
-  for (TableName table : tables) {
-quotaCache.getTableLimiter(table);
-  }
-
-  boolean isUpdated = false;
-  while (!isUpdated) {
-quotaCache.triggerCacheRefresh();
-isUpdated = true;
-for (TableName table : tables) {
-  boolean isBypass = true;
-  if (userLimiter) {
-isBypass = quotaCache.getUserLimiter(User.getCurrent().getUGI(), 
table).isBypass();
+  testUtil.waitFor(6, 250, new ExplainingPredicate() {
+
+@Override
+public boolean evaluate() throw

(hbase) branch branch-2.4 updated: HBASE-28031 TestClusterScopeQuotaThrottle is still failing with broken WAL writer (#5529)(#5539)(#5582)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.4 by this push:
 new 14ac7e0ae3b HBASE-28031 TestClusterScopeQuotaThrottle is still failing 
with broken WAL writer (#5529)(#5539)(#5582)
14ac7e0ae3b is described below

commit 14ac7e0ae3bfb267c9fdc77d084031a1328caa40
Author: Duo Zhang 
AuthorDate: Mon Nov 20 10:41:35 2023 +0800

HBASE-28031 TestClusterScopeQuotaThrottle is still failing with broken WAL 
writer (#5529)(#5539)(#5582)

Limit the scope for EnvironmentEdge injection
Add timeout for ThrottleQuotaTestUtil.triggerCacheRefresh

Signed-off-by: Guanghao Zhang 
Signed-off-by: Bryan Beaudreault 
(cherry picked from commit b1eccb364b30b14bdf475c339dbea12daa6804e2)
---
 .../util/EnvironmentEdgeManagerTestHelper.java |  32 ++
 .../hadoop/hbase/quotas/ThrottleQuotaTestUtil.java | 107 +
 2 files changed, 101 insertions(+), 38 deletions(-)

diff --git 
a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java
 
b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java
index 684247248dc..73e7f1623ef 100644
--- 
a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java
+++ 
b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java
@@ -33,4 +33,36 @@ public final class EnvironmentEdgeManagerTestHelper {
   public static void injectEdge(EnvironmentEdge edge) {
 EnvironmentEdgeManager.injectEdge(edge);
   }
+
+  private static final class PackageEnvironmentEdgeWrapper implements 
EnvironmentEdge {
+
+private final EnvironmentEdge delegate;
+
+private final String packageName;
+
+PackageEnvironmentEdgeWrapper(EnvironmentEdge delegate, String 
packageName) {
+  this.delegate = delegate;
+  this.packageName = packageName;
+}
+
+@Override
+public long currentTime() {
+  StackTraceElement[] elements = new Exception().getStackTrace();
+  // the first element is us, the second one is EnvironmentEdgeManager, so 
let's check the third
+  // one
+  if (elements.length > 2 && 
elements[2].getClassName().startsWith(packageName)) {
+return delegate.currentTime();
+  } else {
+return System.currentTimeMillis();
+  }
+}
+  }
+
+  /**
+   * Inject a {@link EnvironmentEdge} which only takes effect when calling 
directly from the classes
+   * in the given package.
+   */
+  public static void injectEdgeForPackage(EnvironmentEdge edge, String 
packageName) {
+injectEdge(new PackageEnvironmentEdgeWrapper(edge, packageName));
+  }
 }
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
index d349fc25aa1..baf11df848c 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.util.Objects;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Table;
@@ -42,7 +43,9 @@ public final class ThrottleQuotaTestUtil {
   private final static int REFRESH_TIME = 30 * 6;
   static {
 envEdge.setValue(EnvironmentEdgeManager.currentTime());
-EnvironmentEdgeManagerTestHelper.injectEdge(envEdge);
+// only active the envEdge for quotas package
+EnvironmentEdgeManagerTestHelper.injectEdgeForPackage(envEdge,
+  ThrottleQuotaTestUtil.class.getPackage().getName());
   }
 
   private ThrottleQuotaTestUtil() {
@@ -135,51 +138,79 @@ public final class ThrottleQuotaTestUtil {
   RegionServerRpcQuotaManager quotaManager =
 rst.getRegionServer().getRegionServerRpcQuotaManager();
   QuotaCache quotaCache = quotaManager.getQuotaCache();
-
   quotaCache.triggerCacheRefresh();
-  // sleep for cache update
   Thread.sleep(250);
-
-  for (TableName table : tables) {
-quotaCache.getTableLimiter(table);
-  }
-
-  boolean isUpdated = false;
-  while (!isUpdated) {
-quotaCache.triggerCacheRefresh();
-isUpdated = true;
-for (TableName table : tables) {
-  boolean isBypass = true;
-  if (userLimiter) {
-isBypass = quotaCache.getUserLimiter(User.getCurrent().getUGI(), 
table).isBypass();
+  testUtil.waitFor(6, 250, new ExplainingPredicate() {
+
+@Override
+public boolean evaluate() throw

(hbase) branch branch-3 updated: HBASE-28031 TestClusterScopeQuotaThrottle is still failing with broken WAL writer (#5529)(#5539)(#5582)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-3 by this push:
 new b1eccb364b3 HBASE-28031 TestClusterScopeQuotaThrottle is still failing 
with broken WAL writer (#5529)(#5539)(#5582)
b1eccb364b3 is described below

commit b1eccb364b30b14bdf475c339dbea12daa6804e2
Author: Duo Zhang 
AuthorDate: Mon Nov 20 10:41:35 2023 +0800

HBASE-28031 TestClusterScopeQuotaThrottle is still failing with broken WAL 
writer (#5529)(#5539)(#5582)

Limit the scope for EnvironmentEdge injection
Add timeout for ThrottleQuotaTestUtil.triggerCacheRefresh

Signed-off-by: Guanghao Zhang 
Signed-off-by: Bryan Beaudreault 
---
 .../util/EnvironmentEdgeManagerTestHelper.java |  32 ++
 .../hadoop/hbase/quotas/ThrottleQuotaTestUtil.java | 107 +
 2 files changed, 101 insertions(+), 38 deletions(-)

diff --git 
a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java
 
b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java
index 684247248dc..73e7f1623ef 100644
--- 
a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java
+++ 
b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java
@@ -33,4 +33,36 @@ public final class EnvironmentEdgeManagerTestHelper {
   public static void injectEdge(EnvironmentEdge edge) {
 EnvironmentEdgeManager.injectEdge(edge);
   }
+
+  private static final class PackageEnvironmentEdgeWrapper implements 
EnvironmentEdge {
+
+private final EnvironmentEdge delegate;
+
+private final String packageName;
+
+PackageEnvironmentEdgeWrapper(EnvironmentEdge delegate, String 
packageName) {
+  this.delegate = delegate;
+  this.packageName = packageName;
+}
+
+@Override
+public long currentTime() {
+  StackTraceElement[] elements = new Exception().getStackTrace();
+  // the first element is us, the second one is EnvironmentEdgeManager, so 
let's check the third
+  // one
+  if (elements.length > 2 && 
elements[2].getClassName().startsWith(packageName)) {
+return delegate.currentTime();
+  } else {
+return System.currentTimeMillis();
+  }
+}
+  }
+
+  /**
+   * Inject a {@link EnvironmentEdge} which only takes effect when calling 
directly from the classes
+   * in the given package.
+   */
+  public static void injectEdgeForPackage(EnvironmentEdge edge, String 
packageName) {
+injectEdge(new PackageEnvironmentEdgeWrapper(edge, packageName));
+  }
 }
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
index 93eae8dfccf..de6f5653ad2 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.util.Objects;
 import org.apache.hadoop.hbase.HBaseTestingUtil;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Table;
@@ -42,7 +43,9 @@ public final class ThrottleQuotaTestUtil {
   private final static int REFRESH_TIME = 30 * 6;
   static {
 envEdge.setValue(EnvironmentEdgeManager.currentTime());
-EnvironmentEdgeManagerTestHelper.injectEdge(envEdge);
+// only active the envEdge for quotas package
+EnvironmentEdgeManagerTestHelper.injectEdgeForPackage(envEdge,
+  ThrottleQuotaTestUtil.class.getPackage().getName());
   }
 
   private ThrottleQuotaTestUtil() {
@@ -135,51 +138,79 @@ public final class ThrottleQuotaTestUtil {
   RegionServerRpcQuotaManager quotaManager =
 rst.getRegionServer().getRegionServerRpcQuotaManager();
   QuotaCache quotaCache = quotaManager.getQuotaCache();
-
   quotaCache.triggerCacheRefresh();
-  // sleep for cache update
   Thread.sleep(250);
-
-  for (TableName table : tables) {
-quotaCache.getTableLimiter(table);
-  }
-
-  boolean isUpdated = false;
-  while (!isUpdated) {
-quotaCache.triggerCacheRefresh();
-isUpdated = true;
-for (TableName table : tables) {
-  boolean isBypass = true;
-  if (userLimiter) {
-isBypass = quotaCache.getUserLimiter(User.getCurrent().getUGI(), 
table).isBypass();
+  testUtil.waitFor(6, 250, new ExplainingPredicate() {
+
+@Override
+public boolean evaluate() throws Exception {
+  boolean isUpdated = true;
+  for (TableName tab

(hbase) branch master updated: HBASE-28031 Add timeout for ThrottleQuotaTestUtil.triggerCacheRefresh (#5582)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
 new 29cf51d8391 HBASE-28031 Add timeout for 
ThrottleQuotaTestUtil.triggerCacheRefresh (#5582)
29cf51d8391 is described below

commit 29cf51d8391099f5628a52d2c3306932758d4108
Author: Duo Zhang 
AuthorDate: Fri Dec 15 14:13:54 2023 +0800

HBASE-28031 Add timeout for ThrottleQuotaTestUtil.triggerCacheRefresh 
(#5582)

Signed-off-by: Bryan Beaudreault 
---
 .../hadoop/hbase/quotas/ThrottleQuotaTestUtil.java | 103 +
 1 file changed, 66 insertions(+), 37 deletions(-)

diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
index a6e93b663c0..de6f5653ad2 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.util.Objects;
 import org.apache.hadoop.hbase.HBaseTestingUtil;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Table;
@@ -137,51 +138,79 @@ public final class ThrottleQuotaTestUtil {
   RegionServerRpcQuotaManager quotaManager =
 rst.getRegionServer().getRegionServerRpcQuotaManager();
   QuotaCache quotaCache = quotaManager.getQuotaCache();
-
   quotaCache.triggerCacheRefresh();
-  // sleep for cache update
   Thread.sleep(250);
-
-  for (TableName table : tables) {
-quotaCache.getTableLimiter(table);
-  }
-
-  boolean isUpdated = false;
-  while (!isUpdated) {
-quotaCache.triggerCacheRefresh();
-isUpdated = true;
-for (TableName table : tables) {
-  boolean isBypass = true;
-  if (userLimiter) {
-isBypass = quotaCache.getUserLimiter(User.getCurrent().getUGI(), 
table).isBypass();
+  testUtil.waitFor(6, 250, new ExplainingPredicate() {
+
+@Override
+public boolean evaluate() throws Exception {
+  boolean isUpdated = true;
+  for (TableName table : tables) {
+if (userLimiter) {
+  boolean isUserBypass =
+quotaCache.getUserLimiter(User.getCurrent().getUGI(), 
table).isBypass();
+  if (isUserBypass != bypass) {
+LOG.info(
+  "User limiter for user={}, table={} not refreshed, bypass 
expected {}, actual {}",
+  User.getCurrent(), table, bypass, isUserBypass);
+envEdge.incValue(100);
+isUpdated = false;
+break;
+  }
+}
+if (tableLimiter) {
+  boolean isTableBypass = 
quotaCache.getTableLimiter(table).isBypass();
+  if (isTableBypass != bypass) {
+LOG.info("Table limiter for table={} not refreshed, bypass 
expected {}, actual {}",
+  table, bypass, isTableBypass);
+envEdge.incValue(100);
+isUpdated = false;
+break;
+  }
+}
+if (nsLimiter) {
+  boolean isNsBypass =
+
quotaCache.getNamespaceLimiter(table.getNamespaceAsString()).isBypass();
+  if (isNsBypass != bypass) {
+LOG.info(
+  "Namespace limiter for namespace={} not refreshed, bypass 
expected {}, actual {}",
+  table.getNamespaceAsString(), bypass, isNsBypass);
+envEdge.incValue(100);
+isUpdated = false;
+break;
+  }
+}
   }
-  if (tableLimiter) {
-isBypass &= quotaCache.getTableLimiter(table).isBypass();
+  if (rsLimiter) {
+boolean rsIsBypass = quotaCache
+  
.getRegionServerQuotaLimiter(QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY).isBypass();
+if (rsIsBypass != bypass) {
+  LOG.info("RegionServer limiter not refreshed, bypass expected 
{}, actual {}", bypass,
+rsIsBypass);
+  envEdge.incValue(100);
+  isUpdated = false;
+}
   }
-  if (nsLimiter) {
-isBypass &= 
quotaCache.getNamespaceLimiter(table.getNamespaceAsString()).isBypass();
+  if (exceedThrottleQuota) {
+if (quotaCache.isExceedThrottleQuotaEnabled() != bypass) {
+  LOG.info("ExceedThrottleQuotaEnabled not refreshed, bypass 
expected {}, actual {}",
+ 

(hbase) branch branch-3 updated: HBASE-28244 ProcedureTestingUtility.restart is broken sometimes after HBASE-28199 (#5563)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-3 by this push:
 new 0c3a0cfce4a HBASE-28244 ProcedureTestingUtility.restart is broken 
sometimes after HBASE-28199 (#5563)
0c3a0cfce4a is described below

commit 0c3a0cfce4a95a4597d7e79a8fc35bbe784f2fa9
Author: Duo Zhang 
AuthorDate: Wed Dec 13 14:52:12 2023 +0800

HBASE-28244 ProcedureTestingUtility.restart is broken sometimes after 
HBASE-28199 (#5563)

Signed-off-by: Duo Zhang 
(cherry picked from commit 29bfc610d0433f720a34bc47aadca1433bbb1882)
---
 .../hadoop/hbase/procedure2/ProcedureExecutor.java | 46 --
 .../hbase/procedure2/ProcedureFutureUtil.java  | 13 +-
 2 files changed, 46 insertions(+), 13 deletions(-)

diff --git 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
index 5aa11811122..e01a27d7467 100644
--- 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
+++ 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
@@ -35,6 +35,7 @@ import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -605,15 +606,23 @@ public class ProcedureExecutor {
 this.threadGroup = new ThreadGroup("PEWorkerGroup");
 this.timeoutExecutor = new TimeoutExecutorThread<>(this, threadGroup, 
"ProcExecTimeout");
 this.workerMonitorExecutor = new TimeoutExecutorThread<>(this, 
threadGroup, "WorkerMonitor");
+ThreadFactory backingThreadFactory = new ThreadFactory() {
 
+  @Override
+  public Thread newThread(Runnable r) {
+return new Thread(threadGroup, r);
+  }
+};
 int size = Math.max(2, Runtime.getRuntime().availableProcessors());
-ThreadPoolExecutor executor = new ThreadPoolExecutor(size, size, 1, 
TimeUnit.MINUTES,
-  new LinkedBlockingQueue(), new 
ThreadFactoryBuilder().setDaemon(true)
-.setNameFormat(getClass().getSimpleName() + 
"-Async-Task-Executor-%d").build());
+ThreadPoolExecutor executor =
+  new ThreadPoolExecutor(size, size, 1, TimeUnit.MINUTES, new 
LinkedBlockingQueue(),
+new ThreadFactoryBuilder().setDaemon(true)
+  .setNameFormat(getClass().getSimpleName() + 
"-Async-Task-Executor-%d")
+  .setThreadFactory(backingThreadFactory).build());
 executor.allowCoreThreadTimeOut(true);
 this.asyncTaskExecutor = executor;
-forceUpdateExecutor = Executors.newSingleThreadExecutor(
-  new 
ThreadFactoryBuilder().setDaemon(true).setNameFormat("Force-Update-PEWorker-%d").build());
+forceUpdateExecutor = Executors.newFixedThreadPool(1, new 
ThreadFactoryBuilder().setDaemon(true)
+  
.setNameFormat("Force-Update-PEWorker-%d").setThreadFactory(backingThreadFactory).build());
 store.registerListener(new ProcedureStoreListener() {
 
   @Override
@@ -684,10 +693,10 @@ public class ProcedureExecutor {
   }
 
   public void stop() {
-if (!running.getAndSet(false)) {
-  return;
-}
-
+// it is possible that we fail in init, while loading procedures, so we 
will not set running to
+// true but we should have already started the ProcedureScheduler, and 
also the two
+// ExecutorServices, so here we do not check running state, just stop them
+running.set(false);
 LOG.info("Stopping");
 scheduler.stop();
 timeoutExecutor.sendStopSignal();
@@ -708,14 +717,29 @@ public class ProcedureExecutor {
 for (WorkerThread worker : workerThreads) {
   worker.awaitTermination();
 }
+try {
+  if (!forceUpdateExecutor.awaitTermination(5, TimeUnit.SECONDS)) {
+LOG.warn("There are still pending tasks in forceUpdateExecutor");
+  }
+} catch (InterruptedException e) {
+  LOG.warn("interrupted while waiting for forceUpdateExecutor 
termination", e);
+  Thread.currentThread().interrupt();
+}
+try {
+  if (!asyncTaskExecutor.awaitTermination(5, TimeUnit.SECONDS)) {
+LOG.warn("There are still pending tasks in asyncTaskExecutor");
+  }
+} catch (InterruptedException e) {
+  LOG.warn("interrupted while waiting for asyncTaskExecutor termination", 
e);
+  Thread.currentThread().interrupt();
+}
 
 // Destroy the Thread Group for the executors
 // TODO: Fix. #join is not place to dest

(hbase) branch master updated: HBASE-28244 ProcedureTestingUtility.restart is broken sometimes after HBASE-28199 (#5563)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
 new 29bfc610d04 HBASE-28244 ProcedureTestingUtility.restart is broken 
sometimes after HBASE-28199 (#5563)
29bfc610d04 is described below

commit 29bfc610d0433f720a34bc47aadca1433bbb1882
Author: Duo Zhang 
AuthorDate: Wed Dec 13 14:52:12 2023 +0800

HBASE-28244 ProcedureTestingUtility.restart is broken sometimes after 
HBASE-28199 (#5563)

Signed-off-by: Duo Zhang 
---
 .../hadoop/hbase/procedure2/ProcedureExecutor.java | 46 --
 .../hbase/procedure2/ProcedureFutureUtil.java  | 13 +-
 2 files changed, 46 insertions(+), 13 deletions(-)

diff --git 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
index 5aa11811122..e01a27d7467 100644
--- 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
+++ 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
@@ -35,6 +35,7 @@ import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -605,15 +606,23 @@ public class ProcedureExecutor {
 this.threadGroup = new ThreadGroup("PEWorkerGroup");
 this.timeoutExecutor = new TimeoutExecutorThread<>(this, threadGroup, 
"ProcExecTimeout");
 this.workerMonitorExecutor = new TimeoutExecutorThread<>(this, 
threadGroup, "WorkerMonitor");
+ThreadFactory backingThreadFactory = new ThreadFactory() {
 
+  @Override
+  public Thread newThread(Runnable r) {
+return new Thread(threadGroup, r);
+  }
+};
 int size = Math.max(2, Runtime.getRuntime().availableProcessors());
-ThreadPoolExecutor executor = new ThreadPoolExecutor(size, size, 1, 
TimeUnit.MINUTES,
-  new LinkedBlockingQueue(), new 
ThreadFactoryBuilder().setDaemon(true)
-.setNameFormat(getClass().getSimpleName() + 
"-Async-Task-Executor-%d").build());
+ThreadPoolExecutor executor =
+  new ThreadPoolExecutor(size, size, 1, TimeUnit.MINUTES, new 
LinkedBlockingQueue(),
+new ThreadFactoryBuilder().setDaemon(true)
+  .setNameFormat(getClass().getSimpleName() + 
"-Async-Task-Executor-%d")
+  .setThreadFactory(backingThreadFactory).build());
 executor.allowCoreThreadTimeOut(true);
 this.asyncTaskExecutor = executor;
-forceUpdateExecutor = Executors.newSingleThreadExecutor(
-  new 
ThreadFactoryBuilder().setDaemon(true).setNameFormat("Force-Update-PEWorker-%d").build());
+forceUpdateExecutor = Executors.newFixedThreadPool(1, new 
ThreadFactoryBuilder().setDaemon(true)
+  
.setNameFormat("Force-Update-PEWorker-%d").setThreadFactory(backingThreadFactory).build());
 store.registerListener(new ProcedureStoreListener() {
 
   @Override
@@ -684,10 +693,10 @@ public class ProcedureExecutor {
   }
 
   public void stop() {
-if (!running.getAndSet(false)) {
-  return;
-}
-
+// it is possible that we fail in init, while loading procedures, so we 
will not set running to
+// true but we should have already started the ProcedureScheduler, and 
also the two
+// ExecutorServices, so here we do not check running state, just stop them
+running.set(false);
 LOG.info("Stopping");
 scheduler.stop();
 timeoutExecutor.sendStopSignal();
@@ -708,14 +717,29 @@ public class ProcedureExecutor {
 for (WorkerThread worker : workerThreads) {
   worker.awaitTermination();
 }
+try {
+  if (!forceUpdateExecutor.awaitTermination(5, TimeUnit.SECONDS)) {
+LOG.warn("There are still pending tasks in forceUpdateExecutor");
+  }
+} catch (InterruptedException e) {
+  LOG.warn("interrupted while waiting for forceUpdateExecutor 
termination", e);
+  Thread.currentThread().interrupt();
+}
+try {
+  if (!asyncTaskExecutor.awaitTermination(5, TimeUnit.SECONDS)) {
+LOG.warn("There are still pending tasks in asyncTaskExecutor");
+  }
+} catch (InterruptedException e) {
+  LOG.warn("interrupted while waiting for asyncTaskExecutor termination", 
e);
+  Thread.currentThread().interrupt();
+}
 
 // Destroy the Thread Group for the executors
 // TODO: Fix. #join is not place to destroy resources.
 try {
   threadGroup.destroy();
 } c

(hbase) branch branch-2 updated: HBASE-28241 The snapshot operation encountered an NPE and failed. (#5560)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2 by this push:
 new 1d7ea885992 HBASE-28241 The snapshot operation encountered an NPE and 
failed. (#5560)
1d7ea885992 is described below

commit 1d7ea88599244208f79ab8be2ec13a5421622bb9
Author: hiping-tech <58875741+hiping-t...@users.noreply.github.com>
AuthorDate: Tue Dec 12 23:13:01 2023 +0800

HBASE-28241 The snapshot operation encountered an NPE and failed. (#5560)

Fixed the check for an ongoing Snapshot before proceeding with the 
merge/split region operation.

Co-authored-by: lvhaiping.lhp 
Signed-off-by: Duo Zhang 
Signed-off-by: Hui Ruan 
(cherry picked from commit 3d117125892ee36e8a66171fba3a223c09bc0b9a)
---
 .../assignment/MergeTableRegionsProcedure.java |  2 +-
 .../assignment/SplitTableRegionProcedure.java  |  3 +-
 .../assignment/TestMergeTableRegionsProcedure.java | 44 ++
 .../assignment/TestSplitTableRegionProcedure.java  | 53 ++
 .../master/procedure/TestSnapshotProcedure.java| 24 ++
 5 files changed, 124 insertions(+), 2 deletions(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
index 813caa47d33..14f2676d48f 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
@@ -443,7 +443,7 @@ public class MergeTableRegionsProcedure
   private boolean prepareMergeRegion(final MasterProcedureEnv env) throws 
IOException {
 // Fail if we are taking snapshot for the given table
 TableName tn = regionsToMerge[0].getTable();
-if (env.getMasterServices().getSnapshotManager().isTakingSnapshot(tn)) {
+if 
(env.getMasterServices().getSnapshotManager().isTableTakingAnySnapshot(tn)) {
   throw new MergeRegionException("Skip merging regions "
 + RegionInfo.getShortNameToLog(regionsToMerge) + ", because we are 
snapshotting " + tn);
 }
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java
index a0118cbd7b0..2e2182b25d2 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java
@@ -505,7 +505,8 @@ public class SplitTableRegionProcedure
   public boolean prepareSplitRegion(final MasterProcedureEnv env) throws 
IOException {
 // Fail if we are taking snapshot for the given table
 if (
-  
env.getMasterServices().getSnapshotManager().isTakingSnapshot(getParentRegion().getTable())
+  env.getMasterServices().getSnapshotManager()
+.isTableTakingAnySnapshot(getParentRegion().getTable())
 ) {
   setFailure(new IOException("Skip splitting region " + 
getParentRegion().getShortNameToLog()
 + ", because we are taking snapshot for the table " + 
getParentRegion().getTable()));
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java
index 41267a19373..7547b0421b2 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hbase.master.assignment;
 
+import static 
org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.assertProcFailed;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
@@ -33,16 +34,20 @@ import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
+import org.apache.hadoop.hbase.client.SnapshotType;
 import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.client.TableDescriptor;
 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
 import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
 import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
 import org.apache.hadoop.hbase.master.pr

(hbase) branch branch-3 updated: HBASE-28241 The snapshot operation encountered an NPE and failed. (#5560)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-3 by this push:
 new df8ce4a4e0a HBASE-28241 The snapshot operation encountered an NPE and 
failed. (#5560)
df8ce4a4e0a is described below

commit df8ce4a4e0a3953e19eb5eaf5328c10ad85a731d
Author: hiping-tech <58875741+hiping-t...@users.noreply.github.com>
AuthorDate: Tue Dec 12 23:13:01 2023 +0800

HBASE-28241 The snapshot operation encountered an NPE and failed. (#5560)

Fixed the check for an ongoing Snapshot before proceeding with the 
merge/split region operation.

Co-authored-by: lvhaiping.lhp 
Signed-off-by: Duo Zhang 
Signed-off-by: Hui Ruan 
(cherry picked from commit 3d117125892ee36e8a66171fba3a223c09bc0b9a)
---
 .../assignment/MergeTableRegionsProcedure.java |  2 +-
 .../assignment/SplitTableRegionProcedure.java  |  3 +-
 .../assignment/TestMergeTableRegionsProcedure.java | 44 ++
 .../assignment/TestSplitTableRegionProcedure.java  | 53 ++
 .../master/procedure/TestSnapshotProcedure.java| 24 ++
 5 files changed, 124 insertions(+), 2 deletions(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
index c0b47b0bc24..7d4ec71d35b 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
@@ -442,7 +442,7 @@ public class MergeTableRegionsProcedure
   private boolean prepareMergeRegion(final MasterProcedureEnv env) throws 
IOException {
 // Fail if we are taking snapshot for the given table
 TableName tn = regionsToMerge[0].getTable();
-if (env.getMasterServices().getSnapshotManager().isTakingSnapshot(tn)) {
+if 
(env.getMasterServices().getSnapshotManager().isTableTakingAnySnapshot(tn)) {
   throw new MergeRegionException("Skip merging regions "
 + RegionInfo.getShortNameToLog(regionsToMerge) + ", because we are 
snapshotting " + tn);
 }
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java
index a0118cbd7b0..2e2182b25d2 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java
@@ -505,7 +505,8 @@ public class SplitTableRegionProcedure
   public boolean prepareSplitRegion(final MasterProcedureEnv env) throws 
IOException {
 // Fail if we are taking snapshot for the given table
 if (
-  
env.getMasterServices().getSnapshotManager().isTakingSnapshot(getParentRegion().getTable())
+  env.getMasterServices().getSnapshotManager()
+.isTableTakingAnySnapshot(getParentRegion().getTable())
 ) {
   setFailure(new IOException("Skip splitting region " + 
getParentRegion().getShortNameToLog()
 + ", because we are taking snapshot for the table " + 
getParentRegion().getTable()));
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java
index abc6fc45ad3..c0c4e355f2b 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hbase.master.assignment;
 
+import static 
org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.assertProcFailed;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
@@ -33,16 +34,20 @@ import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
+import org.apache.hadoop.hbase.client.SnapshotType;
 import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.client.TableDescriptor;
 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
 import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
 import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
 import org.apache.hadoop.hbase.master.pr

(hbase) branch branch-2.6 updated: HBASE-28241 The snapshot operation encountered an NPE and failed. (#5560)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.6 by this push:
 new 06a9d57439b HBASE-28241 The snapshot operation encountered an NPE and 
failed. (#5560)
06a9d57439b is described below

commit 06a9d57439bd48320f677567227e9d4633635276
Author: hiping-tech <58875741+hiping-t...@users.noreply.github.com>
AuthorDate: Tue Dec 12 23:13:01 2023 +0800

HBASE-28241 The snapshot operation encountered an NPE and failed. (#5560)

Fixed the check for an ongoing Snapshot before proceeding with the 
merge/split region operation.

Co-authored-by: lvhaiping.lhp 
Signed-off-by: Duo Zhang 
Signed-off-by: Hui Ruan 
(cherry picked from commit 3d117125892ee36e8a66171fba3a223c09bc0b9a)
---
 .../assignment/MergeTableRegionsProcedure.java |  2 +-
 .../assignment/SplitTableRegionProcedure.java  |  3 +-
 .../assignment/TestMergeTableRegionsProcedure.java | 44 ++
 .../assignment/TestSplitTableRegionProcedure.java  | 53 ++
 .../master/procedure/TestSnapshotProcedure.java| 24 ++
 5 files changed, 124 insertions(+), 2 deletions(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
index 813caa47d33..14f2676d48f 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
@@ -443,7 +443,7 @@ public class MergeTableRegionsProcedure
   private boolean prepareMergeRegion(final MasterProcedureEnv env) throws 
IOException {
 // Fail if we are taking snapshot for the given table
 TableName tn = regionsToMerge[0].getTable();
-if (env.getMasterServices().getSnapshotManager().isTakingSnapshot(tn)) {
+if 
(env.getMasterServices().getSnapshotManager().isTableTakingAnySnapshot(tn)) {
   throw new MergeRegionException("Skip merging regions "
 + RegionInfo.getShortNameToLog(regionsToMerge) + ", because we are 
snapshotting " + tn);
 }
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java
index a0118cbd7b0..2e2182b25d2 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java
@@ -505,7 +505,8 @@ public class SplitTableRegionProcedure
   public boolean prepareSplitRegion(final MasterProcedureEnv env) throws 
IOException {
 // Fail if we are taking snapshot for the given table
 if (
-  
env.getMasterServices().getSnapshotManager().isTakingSnapshot(getParentRegion().getTable())
+  env.getMasterServices().getSnapshotManager()
+.isTableTakingAnySnapshot(getParentRegion().getTable())
 ) {
   setFailure(new IOException("Skip splitting region " + 
getParentRegion().getShortNameToLog()
 + ", because we are taking snapshot for the table " + 
getParentRegion().getTable()));
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java
index 41267a19373..7547b0421b2 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hbase.master.assignment;
 
+import static 
org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.assertProcFailed;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
@@ -33,16 +34,20 @@ import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
+import org.apache.hadoop.hbase.client.SnapshotType;
 import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.client.TableDescriptor;
 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
 import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
 import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
 import org.apache.hadoop.hbase.master.pr

(hbase) branch master updated: HBASE-28241 The snapshot operation encountered an NPE and failed. (#5560)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
 new 3d117125892 HBASE-28241 The snapshot operation encountered an NPE and 
failed. (#5560)
3d117125892 is described below

commit 3d117125892ee36e8a66171fba3a223c09bc0b9a
Author: hiping-tech <58875741+hiping-t...@users.noreply.github.com>
AuthorDate: Tue Dec 12 23:13:01 2023 +0800

HBASE-28241 The snapshot operation encountered an NPE and failed. (#5560)

Fixed the check for an ongoing Snapshot before proceeding with the 
merge/split region operation.

Co-authored-by: lvhaiping.lhp 
Signed-off-by: Duo Zhang 
Signed-off-by: Hui Ruan 
---
 .../assignment/MergeTableRegionsProcedure.java |  2 +-
 .../assignment/SplitTableRegionProcedure.java  |  3 +-
 .../assignment/TestMergeTableRegionsProcedure.java | 44 ++
 .../assignment/TestSplitTableRegionProcedure.java  | 53 ++
 .../master/procedure/TestSnapshotProcedure.java| 24 ++
 5 files changed, 124 insertions(+), 2 deletions(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
index c0b47b0bc24..7d4ec71d35b 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
@@ -442,7 +442,7 @@ public class MergeTableRegionsProcedure
   private boolean prepareMergeRegion(final MasterProcedureEnv env) throws 
IOException {
 // Fail if we are taking snapshot for the given table
 TableName tn = regionsToMerge[0].getTable();
-if (env.getMasterServices().getSnapshotManager().isTakingSnapshot(tn)) {
+if 
(env.getMasterServices().getSnapshotManager().isTableTakingAnySnapshot(tn)) {
   throw new MergeRegionException("Skip merging regions "
 + RegionInfo.getShortNameToLog(regionsToMerge) + ", because we are 
snapshotting " + tn);
 }
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java
index a0118cbd7b0..2e2182b25d2 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java
@@ -505,7 +505,8 @@ public class SplitTableRegionProcedure
   public boolean prepareSplitRegion(final MasterProcedureEnv env) throws 
IOException {
 // Fail if we are taking snapshot for the given table
 if (
-  
env.getMasterServices().getSnapshotManager().isTakingSnapshot(getParentRegion().getTable())
+  env.getMasterServices().getSnapshotManager()
+.isTableTakingAnySnapshot(getParentRegion().getTable())
 ) {
   setFailure(new IOException("Skip splitting region " + 
getParentRegion().getShortNameToLog()
 + ", because we are taking snapshot for the table " + 
getParentRegion().getTable()));
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java
index abc6fc45ad3..c0c4e355f2b 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hbase.master.assignment;
 
+import static 
org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.assertProcFailed;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
@@ -33,16 +34,20 @@ import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
+import org.apache.hadoop.hbase.client.SnapshotType;
 import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.client.TableDescriptor;
 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
 import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
 import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
 import org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility;
+import org.apache.hadoop.hbase.master.procedure.TestSnapshotProcedu

(hbase) branch branch-3 updated: HBASE-28190 Add slow sync log rolling test in TestAsyncLogRolling. (#5507)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-3 by this push:
 new eede133cad4 HBASE-28190 Add slow sync log rolling test in 
TestAsyncLogRolling. (#5507)
eede133cad4 is described below

commit eede133cad46e8d24b3c796a33f5dd2542452c65
Author: Fantasy-Jay <13631435...@163.com>
AuthorDate: Mon Dec 11 23:22:13 2023 +0800

HBASE-28190 Add slow sync log rolling test in TestAsyncLogRolling. (#5507)

Signed-off-by: Duo Zhang 
(cherry picked from commit 78c5ac372550835133935a3022a0142880476297)
---
 .../hbase/regionserver/wal/AbstractFSWAL.java  |   4 +
 .../hadoop/hbase/regionserver/wal/FSHLog.java  |   8 -
 .../regionserver/wal/AbstractTestLogRolling.java   | 106 +-
 .../regionserver/wal/TestAsyncLogRolling.java  |  65 ++
 .../hbase/regionserver/wal/TestLogRolling.java | 234 +
 5 files changed, 218 insertions(+), 199 deletions(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
index acf3231d4e9..1a5b5384b01 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
@@ -2245,6 +2245,10 @@ public abstract class AbstractFSWAL implements WAL {
 WALSplitter.split(baseDir, p, archiveDir, fs, conf, 
WALFactory.getInstance(conf));
   }
 
+  W getWriter() {
+return this.writer;
+  }
+
   private static void usage() {
 System.err.println("Usage: AbstractFSWAL ");
 System.err.println("Arguments:");
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
index d0d5ce5f2e1..131f284557a 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
@@ -603,14 +603,6 @@ public class FSHLog extends AbstractFSWAL {
 return new DatanodeInfo[0];
   }
 
-  Writer getWriter() {
-return this.writer;
-  }
-
-  void setWriter(Writer writer) {
-this.writer = writer;
-  }
-
   @Override
   protected Writer createCombinedWriter(Writer localWriter, Writer 
remoteWriter) {
 // put remote writer first as usually it will cost more time to finish, so 
we write to it first
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestLogRolling.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestLogRolling.java
index 940dbebf614..2a5aec45882 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestLogRolling.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestLogRolling.java
@@ -20,9 +20,13 @@ package org.apache.hadoop.hbase.regionserver.wal;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hbase.HBaseTestingUtil;
@@ -31,6 +35,7 @@ import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
 import org.apache.hadoop.hbase.StartTestingClusterOption;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.Waiter;
 import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
 import org.apache.hadoop.hbase.client.Get;
@@ -48,8 +53,10 @@ import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
 import org.apache.hadoop.hbase.wal.WAL;
 import org.apache.hadoop.hbase.wal.WALFactory;
+import org.apache.hadoop.hbase.wal.WALProvider;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.junit.After;
+import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -59,6 +66,8 @@ import org.junit.rules.TestName;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import 
org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
+
 /**
  * Test log deletion as logs are rolled.
  */
@@ -74,6 +83,10 @@ public abstract class AbstractTestLogRolling {
   protected static final 

(hbase) branch master updated: HBASE-28190 Add slow sync log rolling test in TestAsyncLogRolling. (#5507)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
 new 78c5ac37255 HBASE-28190 Add slow sync log rolling test in 
TestAsyncLogRolling. (#5507)
78c5ac37255 is described below

commit 78c5ac372550835133935a3022a0142880476297
Author: Fantasy-Jay <13631435...@163.com>
AuthorDate: Mon Dec 11 23:22:13 2023 +0800

HBASE-28190 Add slow sync log rolling test in TestAsyncLogRolling. (#5507)

Signed-off-by: Duo Zhang 
---
 .../hbase/regionserver/wal/AbstractFSWAL.java  |   4 +
 .../hadoop/hbase/regionserver/wal/FSHLog.java  |   8 -
 .../regionserver/wal/AbstractTestLogRolling.java   | 106 +-
 .../regionserver/wal/TestAsyncLogRolling.java  |  65 ++
 .../hbase/regionserver/wal/TestLogRolling.java | 234 +
 5 files changed, 218 insertions(+), 199 deletions(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
index acf3231d4e9..1a5b5384b01 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
@@ -2245,6 +2245,10 @@ public abstract class AbstractFSWAL implements WAL {
 WALSplitter.split(baseDir, p, archiveDir, fs, conf, 
WALFactory.getInstance(conf));
   }
 
+  W getWriter() {
+return this.writer;
+  }
+
   private static void usage() {
 System.err.println("Usage: AbstractFSWAL ");
 System.err.println("Arguments:");
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
index d0d5ce5f2e1..131f284557a 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
@@ -603,14 +603,6 @@ public class FSHLog extends AbstractFSWAL {
 return new DatanodeInfo[0];
   }
 
-  Writer getWriter() {
-return this.writer;
-  }
-
-  void setWriter(Writer writer) {
-this.writer = writer;
-  }
-
   @Override
   protected Writer createCombinedWriter(Writer localWriter, Writer 
remoteWriter) {
 // put remote writer first as usually it will cost more time to finish, so 
we write to it first
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestLogRolling.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestLogRolling.java
index 940dbebf614..2a5aec45882 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestLogRolling.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestLogRolling.java
@@ -20,9 +20,13 @@ package org.apache.hadoop.hbase.regionserver.wal;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hbase.HBaseTestingUtil;
@@ -31,6 +35,7 @@ import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
 import org.apache.hadoop.hbase.StartTestingClusterOption;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.Waiter;
 import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
 import org.apache.hadoop.hbase.client.Get;
@@ -48,8 +53,10 @@ import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
 import org.apache.hadoop.hbase.wal.WAL;
 import org.apache.hadoop.hbase.wal.WALFactory;
+import org.apache.hadoop.hbase.wal.WALProvider;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.junit.After;
+import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -59,6 +66,8 @@ import org.junit.rules.TestName;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import 
org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
+
 /**
  * Test log deletion as logs are rolled.
  */
@@ -74,6 +83,10 @@ public abstract class AbstractTestLogRolling {
   protected static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
   @Rule
   public final Te

(hbase) branch branch-3 updated: HBASE-28248 Race between RegionRemoteProcedureBase and rollback operation could lead to ROLLEDBACK state be persisent to procedure store (#5567)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-3 by this push:
 new 218bde79585 HBASE-28248 Race between RegionRemoteProcedureBase and 
rollback operation could lead to ROLLEDBACK state be persisent to procedure 
store (#5567)
218bde79585 is described below

commit 218bde795850f5969e4ddb1b26e5cc13c3c291ed
Author: Duo Zhang 
AuthorDate: Sat Dec 9 21:55:11 2023 +0800

HBASE-28248 Race between RegionRemoteProcedureBase and rollback operation 
could lead to ROLLEDBACK state be persisent to procedure store (#5567)

Signed-off-by: GeorryHuang 
Signed-off-by: Yi Mei 
(cherry picked from commit 82a2ce10f24a828b2c4960ba85b714a0203c8441)
---
 .../master/assignment/RegionRemoteProcedureBase.java | 16 +++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java
index d27e0068b0c..f6668d9c14b 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RegionRemoteProcedureBaseState;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RegionRemoteProcedureBaseStateData;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
 
 /**
@@ -183,7 +184,20 @@ public abstract class RegionRemoteProcedureBase extends 
Procedure

(hbase) branch branch-2 updated: HBASE-28248 Race between RegionRemoteProcedureBase and rollback operation could lead to ROLLEDBACK state be persisent to procedure store (#5567)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2 by this push:
 new 0d1ffbdf431 HBASE-28248 Race between RegionRemoteProcedureBase and 
rollback operation could lead to ROLLEDBACK state be persisent to procedure 
store (#5567)
0d1ffbdf431 is described below

commit 0d1ffbdf43133326a0ff2fa1719173fecdca0b8a
Author: Duo Zhang 
AuthorDate: Sat Dec 9 21:55:11 2023 +0800

HBASE-28248 Race between RegionRemoteProcedureBase and rollback operation 
could lead to ROLLEDBACK state be persisent to procedure store (#5567)

Signed-off-by: GeorryHuang 
Signed-off-by: Yi Mei 
(cherry picked from commit 82a2ce10f24a828b2c4960ba85b714a0203c8441)
---
 .../master/assignment/RegionRemoteProcedureBase.java | 16 +++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java
index 6b6da9e3396..edfed07e588 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RegionRemoteProcedureBaseState;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RegionRemoteProcedureBaseStateData;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
 
 /**
@@ -179,7 +180,20 @@ public abstract class RegionRemoteProcedureBase extends 
Procedure

(hbase) branch branch-2.6 updated: HBASE-28248 Race between RegionRemoteProcedureBase and rollback operation could lead to ROLLEDBACK state be persisent to procedure store (#5567)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.6 by this push:
 new 276e0830b26 HBASE-28248 Race between RegionRemoteProcedureBase and 
rollback operation could lead to ROLLEDBACK state be persisent to procedure 
store (#5567)
276e0830b26 is described below

commit 276e0830b262522d59fff0d0f3e763df955b8698
Author: Duo Zhang 
AuthorDate: Sat Dec 9 21:55:11 2023 +0800

HBASE-28248 Race between RegionRemoteProcedureBase and rollback operation 
could lead to ROLLEDBACK state be persisent to procedure store (#5567)

Signed-off-by: GeorryHuang 
Signed-off-by: Yi Mei 
(cherry picked from commit 82a2ce10f24a828b2c4960ba85b714a0203c8441)
---
 .../master/assignment/RegionRemoteProcedureBase.java | 16 +++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java
index 6b6da9e3396..edfed07e588 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RegionRemoteProcedureBaseState;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RegionRemoteProcedureBaseStateData;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
 
 /**
@@ -179,7 +180,20 @@ public abstract class RegionRemoteProcedureBase extends 
Procedure

(hbase) branch branch-2.5 updated: HBASE-28248 Race between RegionRemoteProcedureBase and rollback operation could lead to ROLLEDBACK state be persisent to procedure store (#5567)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.5 by this push:
 new a46f7364401 HBASE-28248 Race between RegionRemoteProcedureBase and 
rollback operation could lead to ROLLEDBACK state be persisent to procedure 
store (#5567)
a46f7364401 is described below

commit a46f7364401758f5ad7716db1a13d6b6584f3007
Author: Duo Zhang 
AuthorDate: Sat Dec 9 21:55:11 2023 +0800

HBASE-28248 Race between RegionRemoteProcedureBase and rollback operation 
could lead to ROLLEDBACK state be persisent to procedure store (#5567)

Signed-off-by: GeorryHuang 
Signed-off-by: Yi Mei 
(cherry picked from commit 82a2ce10f24a828b2c4960ba85b714a0203c8441)
---
 .../master/assignment/RegionRemoteProcedureBase.java | 16 +++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java
index 6b6da9e3396..edfed07e588 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RegionRemoteProcedureBaseState;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RegionRemoteProcedureBaseStateData;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
 
 /**
@@ -179,7 +180,20 @@ public abstract class RegionRemoteProcedureBase extends 
Procedure

(hbase) branch branch-2.4 updated: HBASE-28248 Race between RegionRemoteProcedureBase and rollback operation could lead to ROLLEDBACK state be persisent to procedure store (#5567)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.4 by this push:
 new a253472c176 HBASE-28248 Race between RegionRemoteProcedureBase and 
rollback operation could lead to ROLLEDBACK state be persisent to procedure 
store (#5567)
a253472c176 is described below

commit a253472c1764fcf98222f7cf321a83832a2fb680
Author: Duo Zhang 
AuthorDate: Sat Dec 9 21:55:11 2023 +0800

HBASE-28248 Race between RegionRemoteProcedureBase and rollback operation 
could lead to ROLLEDBACK state be persisent to procedure store (#5567)

Signed-off-by: GeorryHuang 
Signed-off-by: Yi Mei 
(cherry picked from commit 82a2ce10f24a828b2c4960ba85b714a0203c8441)
---
 .../master/assignment/RegionRemoteProcedureBase.java | 16 +++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java
index 6b6da9e3396..edfed07e588 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RegionRemoteProcedureBaseState;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RegionRemoteProcedureBaseStateData;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
 
 /**
@@ -179,7 +180,20 @@ public abstract class RegionRemoteProcedureBase extends 
Procedure

(hbase) branch master updated (6e421e9d94a -> 82a2ce10f24)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


from 6e421e9d94a HBASE-28206 [JDK17] JVM crashes intermittently on aarch64 
(#5561)
 add 82a2ce10f24 HBASE-28248 Race between RegionRemoteProcedureBase and 
rollback operation could lead to ROLLEDBACK state be persisent to procedure 
store (#5567)

No new revisions were added by this update.

Summary of changes:
 .../master/assignment/RegionRemoteProcedureBase.java | 16 +++-
 1 file changed, 15 insertions(+), 1 deletion(-)

(hbase) branch branch-3 updated: HBASE-28199 Phase I: Suspend TRSP and SCP when updating meta (#5520)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-3 by this push:
 new 39cc26fe3e3 HBASE-28199 Phase I: Suspend TRSP and SCP when updating 
meta (#5520)
39cc26fe3e3 is described below

commit 39cc26fe3e3710958748aefe60a1606a563e660a
Author: Duo Zhang 
AuthorDate: Mon Dec 4 16:52:57 2023 +0800

HBASE-28199 Phase I: Suspend TRSP and SCP when updating meta (#5520)

Signed-off-by: Yu Li 
(cherry picked from commit cf798adeccd575169a1e1e723cd6e1496c380c3f)
---
 .../hadoop/hbase/procedure2/ProcedureExecutor.java |  57 ---
 .../hbase/master/assignment/AssignmentManager.java | 150 ++-
 .../assignment/RegionRemoteProcedureBase.java  |  46 --
 .../hbase/master/assignment/RegionStateNode.java   |  33 +++-
 .../master/assignment/RegionStateNodeLock.java | 166 +
 .../hbase/master/assignment/RegionStateStore.java  |  67 ++---
 .../assignment/TransitRegionStateProcedure.java| 126 
 .../hbase/master/procedure/MasterProcedureEnv.java |   8 +
 .../master/procedure/ServerCrashProcedure.java |  57 ++-
 .../master/procedure/TruncateRegionProcedure.java  |   2 +-
 ...rateReplicationQueueFromZkToTableProcedure.java |  78 --
 .../master/replication/ReplicationPeerManager.java |   2 +-
 .../hbase/procedure2/ProcedureFutureUtil.java  | 112 ++
 .../master/assignment/MockMasterServices.java  |   4 +-
 .../assignment/TestAssignmentManagerUtil.java  |   3 +-
 .../assignment/TestOpenRegionProcedureBackoff.java |   7 +-
 .../assignment/TestRaceBetweenSCPAndTRSP.java  |  13 +-
 .../master/assignment/TestRegionStateNodeLock.java | 139 +
 .../hbase/master/assignment/TestRollbackSCP.java   |   8 +-
 .../master/procedure/TestProcedurePriority.java|  20 ++-
 20 files changed, 879 insertions(+), 219 deletions(-)

diff --git 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
index 3099c64e00f..5aa11811122 100644
--- 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
+++ 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
@@ -32,8 +32,10 @@ import java.util.PriorityQueue;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -237,6 +239,12 @@ public class ProcedureExecutor {
*/
   private TimeoutExecutorThread workerMonitorExecutor;
 
+  private ExecutorService forceUpdateExecutor;
+
+  // A thread pool for executing some asynchronous tasks for procedures, you 
can find references to
+  // getAsyncTaskExecutor to see the usage
+  private ExecutorService asyncTaskExecutor;
+
   private int corePoolSize;
   private int maxPoolSize;
 
@@ -247,9 +255,6 @@ public class ProcedureExecutor {
*/
   private final ProcedureScheduler scheduler;
 
-  private final Executor forceUpdateExecutor = 
Executors.newSingleThreadExecutor(
-new 
ThreadFactoryBuilder().setDaemon(true).setNameFormat("Force-Update-PEWorker-%d").build());
-
   private final AtomicLong lastProcId = new AtomicLong(-1);
   private final AtomicLong workerId = new AtomicLong(0);
   private final AtomicInteger activeExecutorCount = new AtomicInteger(0);
@@ -317,19 +322,6 @@ public class ProcedureExecutor {
 this.conf = conf;
 this.checkOwnerSet = conf.getBoolean(CHECK_OWNER_SET_CONF_KEY, 
DEFAULT_CHECK_OWNER_SET);
 refreshConfiguration(conf);
-store.registerListener(new ProcedureStoreListener() {
-
-  @Override
-  public void forceUpdate(long[] procIds) {
-Arrays.stream(procIds).forEach(procId -> 
forceUpdateExecutor.execute(() -> {
-  try {
-forceUpdateProcedure(procId);
-  } catch (IOException e) {
-LOG.warn("Failed to force update procedure with pid={}", procId);
-  }
-}));
-  }
-});
   }
 
   private void load(final boolean abortOnCorruption) throws IOException {
@@ -614,6 +606,28 @@ public class ProcedureExecutor {
 this.timeoutExecutor = new TimeoutExecutorThread<>(this, threadGroup, 
"ProcExecTimeout");
 this.workerMonitorExecutor = new TimeoutExecutorThread<>(this, 
threadGroup, "WorkerMonitor");
 
+int size = Math.max(2, Runtime.getRuntime(

(hbase) branch master updated: HBASE-28199 Phase I: Suspend TRSP and SCP when updating meta (#5520)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
 new cf798adeccd HBASE-28199 Phase I: Suspend TRSP and SCP when updating 
meta (#5520)
cf798adeccd is described below

commit cf798adeccd575169a1e1e723cd6e1496c380c3f
Author: Duo Zhang 
AuthorDate: Mon Dec 4 16:52:57 2023 +0800

HBASE-28199 Phase I: Suspend TRSP and SCP when updating meta (#5520)

Signed-off-by: Yu Li 
---
 .../hadoop/hbase/procedure2/ProcedureExecutor.java |  57 ---
 .../hbase/master/assignment/AssignmentManager.java | 150 ++-
 .../assignment/RegionRemoteProcedureBase.java  |  46 --
 .../hbase/master/assignment/RegionStateNode.java   |  33 +++-
 .../master/assignment/RegionStateNodeLock.java | 166 +
 .../hbase/master/assignment/RegionStateStore.java  |  67 ++---
 .../assignment/TransitRegionStateProcedure.java| 126 
 .../hbase/master/procedure/MasterProcedureEnv.java |   8 +
 .../master/procedure/ServerCrashProcedure.java |  57 ++-
 .../master/procedure/TruncateRegionProcedure.java  |   2 +-
 ...rateReplicationQueueFromZkToTableProcedure.java |  78 --
 .../master/replication/ReplicationPeerManager.java |   2 +-
 .../hbase/procedure2/ProcedureFutureUtil.java  | 112 ++
 .../master/assignment/MockMasterServices.java  |   4 +-
 .../assignment/TestAssignmentManagerUtil.java  |   3 +-
 .../assignment/TestOpenRegionProcedureBackoff.java |   7 +-
 .../assignment/TestRaceBetweenSCPAndTRSP.java  |  13 +-
 .../master/assignment/TestRegionStateNodeLock.java | 139 +
 .../hbase/master/assignment/TestRollbackSCP.java   |   8 +-
 .../master/procedure/TestProcedurePriority.java|  20 ++-
 20 files changed, 879 insertions(+), 219 deletions(-)

diff --git 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
index 3099c64e00f..5aa11811122 100644
--- 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
+++ 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
@@ -32,8 +32,10 @@ import java.util.PriorityQueue;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -237,6 +239,12 @@ public class ProcedureExecutor {
*/
   private TimeoutExecutorThread workerMonitorExecutor;
 
+  private ExecutorService forceUpdateExecutor;
+
+  // A thread pool for executing some asynchronous tasks for procedures, you 
can find references to
+  // getAsyncTaskExecutor to see the usage
+  private ExecutorService asyncTaskExecutor;
+
   private int corePoolSize;
   private int maxPoolSize;
 
@@ -247,9 +255,6 @@ public class ProcedureExecutor {
*/
   private final ProcedureScheduler scheduler;
 
-  private final Executor forceUpdateExecutor = 
Executors.newSingleThreadExecutor(
-new 
ThreadFactoryBuilder().setDaemon(true).setNameFormat("Force-Update-PEWorker-%d").build());
-
   private final AtomicLong lastProcId = new AtomicLong(-1);
   private final AtomicLong workerId = new AtomicLong(0);
   private final AtomicInteger activeExecutorCount = new AtomicInteger(0);
@@ -317,19 +322,6 @@ public class ProcedureExecutor {
 this.conf = conf;
 this.checkOwnerSet = conf.getBoolean(CHECK_OWNER_SET_CONF_KEY, 
DEFAULT_CHECK_OWNER_SET);
 refreshConfiguration(conf);
-store.registerListener(new ProcedureStoreListener() {
-
-  @Override
-  public void forceUpdate(long[] procIds) {
-Arrays.stream(procIds).forEach(procId -> 
forceUpdateExecutor.execute(() -> {
-  try {
-forceUpdateProcedure(procId);
-  } catch (IOException e) {
-LOG.warn("Failed to force update procedure with pid={}", procId);
-  }
-}));
-  }
-});
   }
 
   private void load(final boolean abortOnCorruption) throws IOException {
@@ -614,6 +606,28 @@ public class ProcedureExecutor {
 this.timeoutExecutor = new TimeoutExecutorThread<>(this, threadGroup, 
"ProcExecTimeout");
 this.workerMonitorExecutor = new TimeoutExecutorThread<>(this, 
threadGroup, "WorkerMonitor");
 
+int size = Math.max(2, Runtime.getRuntime().availableProcessors());
+ThreadPoolExecutor executor = new Threa

(hbase) branch branch-3 updated: HBASE-28226 Add logic to check for RegionStateNode null pointer in FlushRegionProcedure (#5548)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-3 by this push:
 new f6d96dc68ac HBASE-28226 Add logic to check for RegionStateNode null 
pointer in FlushRegionProcedure (#5548)
f6d96dc68ac is described below

commit f6d96dc68ac4b2dd79e6c67fac5c45daa98ee581
Author: hiping-tech <58875741+hiping-t...@users.noreply.github.com>
AuthorDate: Mon Dec 4 15:51:28 2023 +0800

HBASE-28226 Add logic to check for RegionStateNode null pointer in 
FlushRegionProcedure (#5548)

Co-authored-by: lvhaiping.lhp 
Signed-off-by: Duo Zhang 
(cherry picked from commit 8631714705231aa5db5397d92e4ea9d0e5625129)
---
 .../apache/hadoop/hbase/master/procedure/FlushRegionProcedure.java   | 5 +
 1 file changed, 5 insertions(+)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/FlushRegionProcedure.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/FlushRegionProcedure.java
index 67f0442b618..88f7e652cbf 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/FlushRegionProcedure.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/FlushRegionProcedure.java
@@ -88,6 +88,11 @@ public class FlushRegionProcedure extends 
Procedure
 
 RegionStates regionStates = env.getAssignmentManager().getRegionStates();
 RegionStateNode regionNode = regionStates.getRegionStateNode(region);
+if (regionNode == null) {
+  LOG.debug("Region {} is not in region states, it is very likely that it 
has been cleared by"
++ " other procedures such as merge or split, so skip {}. See 
HBASE-28226", region, this);
+  return null;
+}
 regionNode.lock();
 try {
   if (!regionNode.isInState(State.OPEN) || regionNode.isInTransition()) {

(hbase) branch branch-2 updated: HBASE-28226 Add logic to check for RegionStateNode null pointer in FlushRegionProcedure (#5548)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2 by this push:
 new 3f485560577 HBASE-28226 Add logic to check for RegionStateNode null 
pointer in FlushRegionProcedure (#5548)
3f485560577 is described below

commit 3f485560577941d0728192dd1b2562398e096450
Author: hiping-tech <58875741+hiping-t...@users.noreply.github.com>
AuthorDate: Mon Dec 4 15:51:28 2023 +0800

HBASE-28226 Add logic to check for RegionStateNode null pointer in 
FlushRegionProcedure (#5548)

Co-authored-by: lvhaiping.lhp 
Signed-off-by: Duo Zhang 
(cherry picked from commit 8631714705231aa5db5397d92e4ea9d0e5625129)
---
 .../apache/hadoop/hbase/master/procedure/FlushRegionProcedure.java   | 5 +
 1 file changed, 5 insertions(+)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/FlushRegionProcedure.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/FlushRegionProcedure.java
index 67f0442b618..88f7e652cbf 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/FlushRegionProcedure.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/FlushRegionProcedure.java
@@ -88,6 +88,11 @@ public class FlushRegionProcedure extends 
Procedure
 
 RegionStates regionStates = env.getAssignmentManager().getRegionStates();
 RegionStateNode regionNode = regionStates.getRegionStateNode(region);
+if (regionNode == null) {
+  LOG.debug("Region {} is not in region states, it is very likely that it 
has been cleared by"
++ " other procedures such as merge or split, so skip {}. See 
HBASE-28226", region, this);
+  return null;
+}
 regionNode.lock();
 try {
   if (!regionNode.isInState(State.OPEN) || regionNode.isInTransition()) {

(hbase) branch branch-2.6 updated: HBASE-28226 Add logic to check for RegionStateNode null pointer in FlushRegionProcedure (#5548)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.6 by this push:
 new 35603fa1d49 HBASE-28226 Add logic to check for RegionStateNode null 
pointer in FlushRegionProcedure (#5548)
35603fa1d49 is described below

commit 35603fa1d49c416d9300bf0472147b3fb1f3bbb8
Author: hiping-tech <58875741+hiping-t...@users.noreply.github.com>
AuthorDate: Mon Dec 4 15:51:28 2023 +0800

HBASE-28226 Add logic to check for RegionStateNode null pointer in 
FlushRegionProcedure (#5548)

Co-authored-by: lvhaiping.lhp 
Signed-off-by: Duo Zhang 
(cherry picked from commit 8631714705231aa5db5397d92e4ea9d0e5625129)
---
 .../apache/hadoop/hbase/master/procedure/FlushRegionProcedure.java   | 5 +
 1 file changed, 5 insertions(+)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/FlushRegionProcedure.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/FlushRegionProcedure.java
index 67f0442b618..88f7e652cbf 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/FlushRegionProcedure.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/FlushRegionProcedure.java
@@ -88,6 +88,11 @@ public class FlushRegionProcedure extends 
Procedure
 
 RegionStates regionStates = env.getAssignmentManager().getRegionStates();
 RegionStateNode regionNode = regionStates.getRegionStateNode(region);
+if (regionNode == null) {
+  LOG.debug("Region {} is not in region states, it is very likely that it 
has been cleared by"
++ " other procedures such as merge or split, so skip {}. See 
HBASE-28226", region, this);
+  return null;
+}
 regionNode.lock();
 try {
   if (!regionNode.isInState(State.OPEN) || regionNode.isInTransition()) {

(hbase) branch master updated: HBASE-28226 Add logic to check for RegionStateNode null pointer in FlushRegionProcedure (#5548)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
 new 86317147052 HBASE-28226 Add logic to check for RegionStateNode null 
pointer in FlushRegionProcedure (#5548)
86317147052 is described below

commit 8631714705231aa5db5397d92e4ea9d0e5625129
Author: hiping-tech <58875741+hiping-t...@users.noreply.github.com>
AuthorDate: Mon Dec 4 15:51:28 2023 +0800

HBASE-28226 Add logic to check for RegionStateNode null pointer in 
FlushRegionProcedure (#5548)

Co-authored-by: lvhaiping.lhp 
Signed-off-by: Duo Zhang 
---
 .../apache/hadoop/hbase/master/procedure/FlushRegionProcedure.java   | 5 +
 1 file changed, 5 insertions(+)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/FlushRegionProcedure.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/FlushRegionProcedure.java
index 67f0442b618..88f7e652cbf 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/FlushRegionProcedure.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/FlushRegionProcedure.java
@@ -88,6 +88,11 @@ public class FlushRegionProcedure extends 
Procedure
 
 RegionStates regionStates = env.getAssignmentManager().getRegionStates();
 RegionStateNode regionNode = regionStates.getRegionStateNode(region);
+if (regionNode == null) {
+  LOG.debug("Region {} is not in region states, it is very likely that it 
has been cleared by"
++ " other procedures such as merge or split, so skip {}. See 
HBASE-28226", region, this);
+  return null;
+}
 regionNode.lock();
 try {
   if (!regionNode.isInState(State.OPEN) || regionNode.isInTransition()) {

(hbase) branch branch-2.6 updated: HBASE-28212 Addendum fix TestShell (#5555)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.6 by this push:
 new 23e9b54538b HBASE-28212 Addendum fix TestShell (#)
23e9b54538b is described below

commit 23e9b54538b5916bfc6acf769685643f26e59e40
Author: Duo Zhang 
AuthorDate: Fri Dec 1 23:31:33 2023 +0800

HBASE-28212 Addendum fix TestShell (#)

We added a new field in Procedure so the json output is also changed
thus we need to change the assertion

(cherry picked from commit 7dd4d0c532a0d4e60037c193b2ce22f0b03ca01e)
---
 hbase-shell/src/test/ruby/shell/list_locks_test.rb | 19 ---
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/hbase-shell/src/test/ruby/shell/list_locks_test.rb 
b/hbase-shell/src/test/ruby/shell/list_locks_test.rb
index 100c145a3ac..7013b65e0f9 100644
--- a/hbase-shell/src/test/ruby/shell/list_locks_test.rb
+++ b/hbase-shell/src/test/ruby/shell/list_locks_test.rb
@@ -81,7 +81,8 @@ module Hbase
   
"\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
   "\"procId\"=>\"0\", \"submittedTime\"=>\"0\", 
\"state\"=>\"RUNNABLE\", " \
   "\"lastUpdate\"=>\"0\", " \
-  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}]" \
+  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}], " \
+  "\"executed\"=>false" \
 "}\n\n",
 output)
 end
@@ -101,7 +102,8 @@ module Hbase
   
"\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
   "\"procId\"=>\"0\", \"submittedTime\"=>\"0\", 
\"state\"=>\"RUNNABLE\", " \
   "\"lastUpdate\"=>\"0\", " \
-  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}]" \
+  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}], " \
+  "\"executed\"=>false" \
 "}\n\n",
 output)
 end
@@ -119,7 +121,8 @@ module Hbase
   
"\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
   "\"procId\"=>\"1\", \"submittedTime\"=>\"0\", 
\"state\"=>\"RUNNABLE\", " \
   "\"lastUpdate\"=>\"0\", " \
-  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}]" \
+  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}], " \
+  "\"executed\"=>false" \
 "}\n\n" \
 "TABLE(hbase:namespace)\n" \
 "Lock type: SHARED, count: 1\n\n",
@@ -143,7 +146,8 @@ module Hbase
   
"\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
   "\"procId\"=>\"2\", \"submittedTime\"=>\"0\", 
\"state\"=>\"RUNNABLE\", " \
   "\"lastUpdate\"=>\"0\", " \
-  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}]" \
+  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}], " \
+  "\"executed\"=>false" \
 "}\n\n",
 output)
 end
@@ -168,7 +172,8 @@ module Hbase
   
"\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
   "\"procId\"=>\"3\", \"submittedTime\"=>\"0\", 
\"state\"=>\"RUNNABLE\", " \
   "\"lastUpdate\"=>\"0\", " \
-  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}]" \
+  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}], " \
+  "\"executed\"=>false" \
 "}\n\n",
 output)
 end
@@ -198,14 +203,14 @@ module Hbase
   "\"lastUpdate\"=>\"0\", \"stateMessage\"=>[{" \
 "\"lockType\"=>\"EXCLUSIVE\", " \
 "\"tableName\"=>{\"namespace\"=>\"bnM0\", 
\"qualifier\"=>\"dGFibGU0\"" \
-  "}, \"description\"=>\"description\"}]}\n" \
+  "}, \"description\"=>\"description\"}], \"executed\"=>false}\n" \
 "Waiting procedures\n" \
 
"{\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
   "\"procId\"=>\"2\", \"submittedTime\"=>\"0\", 
\"state\"=>\"RUNNABLE\", " \
   "\"lastUpdate\"=>\"0\", \"stateMessage\"=>[{" \
 "\"lockType\"=>\"SHARED\", " \
 "\"tableName\"=>{\"namespace\"=>\"bnM0\", 
\"qualifier\"=>\"dGFibGU0\"}, " \
-"\"description\"=>\"description\"}]}\n" \
+"\"description\"=>\"description\"}], \"executed\"=>false}\n" \
 "1 row(s)\n\n",
 output)
 end

(hbase) branch branch-2 updated (0f34724486c -> dbc165a3cf3)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a change to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git


from 0f34724486c HBASE-28222 Leak in ExportSnapshot during verifySnapshot 
on S3A (#5554)
 add dbc165a3cf3 HBASE-28212 Addendum fix TestShell (#)

No new revisions were added by this update.

Summary of changes:
 hbase-shell/src/test/ruby/shell/list_locks_test.rb | 19 ---
 1 file changed, 12 insertions(+), 7 deletions(-)

(hbase) branch branch-3 updated: HBASE-28212 Addendum fix TestShell (#5555)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-3 by this push:
 new 3b72456b0b9 HBASE-28212 Addendum fix TestShell (#)
3b72456b0b9 is described below

commit 3b72456b0b96cb82eb403a3779e4876b9a05f87c
Author: Duo Zhang 
AuthorDate: Fri Dec 1 23:31:33 2023 +0800

HBASE-28212 Addendum fix TestShell (#)

We added a new field in Procedure so the json output is also changed
thus we need to change the assertion

(cherry picked from commit 7dd4d0c532a0d4e60037c193b2ce22f0b03ca01e)
---
 hbase-shell/src/test/ruby/shell/list_locks_test.rb | 19 ---
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/hbase-shell/src/test/ruby/shell/list_locks_test.rb 
b/hbase-shell/src/test/ruby/shell/list_locks_test.rb
index 20a910c485d..89c6940db2a 100644
--- a/hbase-shell/src/test/ruby/shell/list_locks_test.rb
+++ b/hbase-shell/src/test/ruby/shell/list_locks_test.rb
@@ -81,7 +81,8 @@ module Hbase
   
"\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
   "\"procId\"=>\"0\", \"submittedTime\"=>\"0\", 
\"state\"=>\"RUNNABLE\", " \
   "\"lastUpdate\"=>\"0\", " \
-  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}]" \
+  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}], " \
+  "\"executed\"=>false" \
 "}\n\n",
 output)
 end
@@ -101,7 +102,8 @@ module Hbase
   
"\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
   "\"procId\"=>\"0\", \"submittedTime\"=>\"0\", 
\"state\"=>\"RUNNABLE\", " \
   "\"lastUpdate\"=>\"0\", " \
-  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}]" \
+  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}], " \
+  "\"executed\"=>false" \
 "}\n\n",
 output)
 end
@@ -119,7 +121,8 @@ module Hbase
   
"\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
   "\"procId\"=>\"1\", \"submittedTime\"=>\"0\", 
\"state\"=>\"RUNNABLE\", " \
   "\"lastUpdate\"=>\"0\", " \
-  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}]" \
+  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}], " \
+  "\"executed\"=>false" \
 "}\n\n" \
 "TABLE(hbase:namespace)\n" \
 "Lock type: SHARED, count: 1\n\n",
@@ -143,7 +146,8 @@ module Hbase
   
"\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
   "\"procId\"=>\"2\", \"submittedTime\"=>\"0\", 
\"state\"=>\"RUNNABLE\", " \
   "\"lastUpdate\"=>\"0\", " \
-  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}]" \
+  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}], " \
+  "\"executed\"=>false" \
 "}\n\n",
 output)
 end
@@ -168,7 +172,8 @@ module Hbase
   
"\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
   "\"procId\"=>\"3\", \"submittedTime\"=>\"0\", 
\"state\"=>\"RUNNABLE\", " \
   "\"lastUpdate\"=>\"0\", " \
-  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}]" \
+  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}], " \
+  "\"executed\"=>false" \
 "}\n\n",
 output)
 end
@@ -198,14 +203,14 @@ module Hbase
   "\"lastUpdate\"=>\"0\", \"stateMessage\"=>[{" \
 "\"lockType\"=>\"EXCLUSIVE\", " \
 "\"tableName\"=>{\"namespace\"=>\"bnM0\", 
\"qualifier\"=>\"dGFibGU0\"" \
-  "}, \"description\"=>\"description\"}]}\n" \
+  "}, \"description\"=>\"description\"}], \"executed\"=>false}\n" \
 "Waiting procedures\n" \
 
"{\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
   "\"procId\"=>\"2\", \"submittedTime\"=>\"0\", 
\"state\"=>\"RUNNABLE\", " \
   "\"lastUpdate\"=>\"0\", \"stateMessage\"=>[{" \
 "\"lockType\"=>\"SHARED\", " \
 "\"tableName\"=>{\"namespace\"=>\"bnM0\", 
\"qualifier\"=>\"dGFibGU0\"}, " \
-"\"description\"=>\"description\"}]}\n" \
+"\"description\"=>\"description\"}], \"executed\"=>false}\n" \
 "1 row(s)\n\n",
 output)
 end

(hbase) branch branch-2.5 updated: HBASE-28212 Addendum fix TestShell (#5555)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.5 by this push:
 new 4e3c202535f HBASE-28212 Addendum fix TestShell (#)
4e3c202535f is described below

commit 4e3c202535f64fd39ec835297dcea1b87da93bb1
Author: Duo Zhang 
AuthorDate: Fri Dec 1 23:31:33 2023 +0800

HBASE-28212 Addendum fix TestShell (#)

We added a new field in Procedure so the json output is also changed
thus we need to change the assertion

(cherry picked from commit 7dd4d0c532a0d4e60037c193b2ce22f0b03ca01e)
---
 hbase-shell/src/test/ruby/shell/list_locks_test.rb | 19 ---
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/hbase-shell/src/test/ruby/shell/list_locks_test.rb 
b/hbase-shell/src/test/ruby/shell/list_locks_test.rb
index 100c145a3ac..7013b65e0f9 100644
--- a/hbase-shell/src/test/ruby/shell/list_locks_test.rb
+++ b/hbase-shell/src/test/ruby/shell/list_locks_test.rb
@@ -81,7 +81,8 @@ module Hbase
   
"\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
   "\"procId\"=>\"0\", \"submittedTime\"=>\"0\", 
\"state\"=>\"RUNNABLE\", " \
   "\"lastUpdate\"=>\"0\", " \
-  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}]" \
+  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}], " \
+  "\"executed\"=>false" \
 "}\n\n",
 output)
 end
@@ -101,7 +102,8 @@ module Hbase
   
"\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
   "\"procId\"=>\"0\", \"submittedTime\"=>\"0\", 
\"state\"=>\"RUNNABLE\", " \
   "\"lastUpdate\"=>\"0\", " \
-  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}]" \
+  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}], " \
+  "\"executed\"=>false" \
 "}\n\n",
 output)
 end
@@ -119,7 +121,8 @@ module Hbase
   
"\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
   "\"procId\"=>\"1\", \"submittedTime\"=>\"0\", 
\"state\"=>\"RUNNABLE\", " \
   "\"lastUpdate\"=>\"0\", " \
-  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}]" \
+  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}], " \
+  "\"executed\"=>false" \
 "}\n\n" \
 "TABLE(hbase:namespace)\n" \
 "Lock type: SHARED, count: 1\n\n",
@@ -143,7 +146,8 @@ module Hbase
   
"\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
   "\"procId\"=>\"2\", \"submittedTime\"=>\"0\", 
\"state\"=>\"RUNNABLE\", " \
   "\"lastUpdate\"=>\"0\", " \
-  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}]" \
+  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}], " \
+  "\"executed\"=>false" \
 "}\n\n",
 output)
 end
@@ -168,7 +172,8 @@ module Hbase
   
"\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
   "\"procId\"=>\"3\", \"submittedTime\"=>\"0\", 
\"state\"=>\"RUNNABLE\", " \
   "\"lastUpdate\"=>\"0\", " \
-  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}]" \
+  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}], " \
+  "\"executed\"=>false" \
 "}\n\n",
 output)
 end
@@ -198,14 +203,14 @@ module Hbase
   "\"lastUpdate\"=>\"0\", \"stateMessage\"=>[{" \
 "\"lockType\"=>\"EXCLUSIVE\", " \
 "\"tableName\"=>{\"namespace\"=>\"bnM0\", 
\"qualifier\"=>\"dGFibGU0\"" \
-  "}, \"description\"=>\"description\"}]}\n" \
+  "}, \"description\"=>\"description\"}], \"executed\"=>false}\n" \
 "Waiting procedures\n" \
 
"{\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
   "\"procId\"=>\"2\", \"submittedTime\"=>\"0\", 
\"state\"=>\"RUNNABLE\", " \
   "\"lastUpdate\"=>\"0\", \"stateMessage\"=>[{" \
 "\"lockType\"=>\"SHARED\", " \
 "\"tableName\"=>{\"namespace\"=>\"bnM0\", 
\"qualifier\"=>\"dGFibGU0\"}, " \
-"\"description\"=>\"description\"}]}\n" \
+"\"description\"=>\"description\"}], \"executed\"=>false}\n" \
 "1 row(s)\n\n",
 output)
 end

(hbase) branch branch-2.4 updated: HBASE-28212 Addendum fix TestShell (#5555)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.4 by this push:
 new 808249182d7 HBASE-28212 Addendum fix TestShell (#)
808249182d7 is described below

commit 808249182d760cc7365f527e70cef212faac3a23
Author: Duo Zhang 
AuthorDate: Fri Dec 1 23:31:33 2023 +0800

HBASE-28212 Addendum fix TestShell (#)

We added a new field in Procedure so the json output is also changed
thus we need to change the assertion

(cherry picked from commit 7dd4d0c532a0d4e60037c193b2ce22f0b03ca01e)
---
 hbase-shell/src/test/ruby/shell/list_locks_test.rb | 19 ---
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/hbase-shell/src/test/ruby/shell/list_locks_test.rb 
b/hbase-shell/src/test/ruby/shell/list_locks_test.rb
index 100c145a3ac..7013b65e0f9 100644
--- a/hbase-shell/src/test/ruby/shell/list_locks_test.rb
+++ b/hbase-shell/src/test/ruby/shell/list_locks_test.rb
@@ -81,7 +81,8 @@ module Hbase
   
"\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
   "\"procId\"=>\"0\", \"submittedTime\"=>\"0\", 
\"state\"=>\"RUNNABLE\", " \
   "\"lastUpdate\"=>\"0\", " \
-  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}]" \
+  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}], " \
+  "\"executed\"=>false" \
 "}\n\n",
 output)
 end
@@ -101,7 +102,8 @@ module Hbase
   
"\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
   "\"procId\"=>\"0\", \"submittedTime\"=>\"0\", 
\"state\"=>\"RUNNABLE\", " \
   "\"lastUpdate\"=>\"0\", " \
-  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}]" \
+  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}], " \
+  "\"executed\"=>false" \
 "}\n\n",
 output)
 end
@@ -119,7 +121,8 @@ module Hbase
   
"\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
   "\"procId\"=>\"1\", \"submittedTime\"=>\"0\", 
\"state\"=>\"RUNNABLE\", " \
   "\"lastUpdate\"=>\"0\", " \
-  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}]" \
+  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}], " \
+  "\"executed\"=>false" \
 "}\n\n" \
 "TABLE(hbase:namespace)\n" \
 "Lock type: SHARED, count: 1\n\n",
@@ -143,7 +146,8 @@ module Hbase
   
"\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
   "\"procId\"=>\"2\", \"submittedTime\"=>\"0\", 
\"state\"=>\"RUNNABLE\", " \
   "\"lastUpdate\"=>\"0\", " \
-  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}]" \
+  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}], " \
+  "\"executed\"=>false" \
 "}\n\n",
 output)
 end
@@ -168,7 +172,8 @@ module Hbase
   
"\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
   "\"procId\"=>\"3\", \"submittedTime\"=>\"0\", 
\"state\"=>\"RUNNABLE\", " \
   "\"lastUpdate\"=>\"0\", " \
-  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}]" \
+  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}], " \
+  "\"executed\"=>false" \
 "}\n\n",
 output)
 end
@@ -198,14 +203,14 @@ module Hbase
   "\"lastUpdate\"=>\"0\", \"stateMessage\"=>[{" \
 "\"lockType\"=>\"EXCLUSIVE\", " \
 "\"tableName\"=>{\"namespace\"=>\"bnM0\", 
\"qualifier\"=>\"dGFibGU0\"" \
-  "}, \"description\"=>\"description\"}]}\n" \
+  "}, \"description\"=>\"description\"}], \"executed\"=>false}\n" \
 "Waiting procedures\n" \
 
"{\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
   "\"procId\"=>\"2\", \"submittedTime\"=>\"0\", 
\"state\"=>\"RUNNABLE\", " \
   "\"lastUpdate\"=>\"0\", \"stateMessage\"=>[{" \
 "\"lockType\"=>\"SHARED\", " \
 "\"tableName\"=>{\"namespace\"=>\"bnM0\", 
\"qualifier\"=>\"dGFibGU0\"}, " \
-"\"description\"=>\"description\"}]}\n" \
+"\"description\"=>\"description\"}], \"executed\"=>false}\n" \
 "1 row(s)\n\n",
 output)
 end

(hbase) branch master updated: HBASE-28212 Addendum fix TestShell (#5555)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
 new 7dd4d0c532a HBASE-28212 Addendum fix TestShell (#)
7dd4d0c532a is described below

commit 7dd4d0c532a0d4e60037c193b2ce22f0b03ca01e
Author: Duo Zhang 
AuthorDate: Fri Dec 1 23:31:33 2023 +0800

HBASE-28212 Addendum fix TestShell (#)

We added a new field in Procedure so the json output is also changed
thus we need to change the assertion
---
 hbase-shell/src/test/ruby/shell/list_locks_test.rb | 19 ---
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/hbase-shell/src/test/ruby/shell/list_locks_test.rb 
b/hbase-shell/src/test/ruby/shell/list_locks_test.rb
index 20a910c485d..89c6940db2a 100644
--- a/hbase-shell/src/test/ruby/shell/list_locks_test.rb
+++ b/hbase-shell/src/test/ruby/shell/list_locks_test.rb
@@ -81,7 +81,8 @@ module Hbase
   
"\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
   "\"procId\"=>\"0\", \"submittedTime\"=>\"0\", 
\"state\"=>\"RUNNABLE\", " \
   "\"lastUpdate\"=>\"0\", " \
-  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}]" \
+  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}], " \
+  "\"executed\"=>false" \
 "}\n\n",
 output)
 end
@@ -101,7 +102,8 @@ module Hbase
   
"\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
   "\"procId\"=>\"0\", \"submittedTime\"=>\"0\", 
\"state\"=>\"RUNNABLE\", " \
   "\"lastUpdate\"=>\"0\", " \
-  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}]" \
+  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}], " \
+  "\"executed\"=>false" \
 "}\n\n",
 output)
 end
@@ -119,7 +121,8 @@ module Hbase
   
"\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
   "\"procId\"=>\"1\", \"submittedTime\"=>\"0\", 
\"state\"=>\"RUNNABLE\", " \
   "\"lastUpdate\"=>\"0\", " \
-  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}]" \
+  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}], " \
+  "\"executed\"=>false" \
 "}\n\n" \
 "TABLE(hbase:namespace)\n" \
 "Lock type: SHARED, count: 1\n\n",
@@ -143,7 +146,8 @@ module Hbase
   
"\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
   "\"procId\"=>\"2\", \"submittedTime\"=>\"0\", 
\"state\"=>\"RUNNABLE\", " \
   "\"lastUpdate\"=>\"0\", " \
-  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}]" \
+  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}], " \
+  "\"executed\"=>false" \
 "}\n\n",
 output)
 end
@@ -168,7 +172,8 @@ module Hbase
   
"\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
   "\"procId\"=>\"3\", \"submittedTime\"=>\"0\", 
\"state\"=>\"RUNNABLE\", " \
   "\"lastUpdate\"=>\"0\", " \
-  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}]" \
+  "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", 
\"description\"=>\"description\"}], " \
+  "\"executed\"=>false" \
 "}\n\n",
 output)
 end
@@ -198,14 +203,14 @@ module Hbase
   "\"lastUpdate\"=>\"0\", \"stateMessage\"=>[{" \
 "\"lockType\"=>\"EXCLUSIVE\", " \
 "\"tableName\"=>{\"namespace\"=>\"bnM0\", 
\"qualifier\"=>\"dGFibGU0\"" \
-  "}, \"description\"=>\"description\"}]}\n" \
+  "}, \"description\"=>\"description\"}], \"executed\"=>false}\n" \
 "Waiting procedures\n" \
 
"{\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
   "\"procId\"=>\"2\", \"submittedTime\"=>\"0\", 
\"state\"=>\"RUNNABLE\", " \
   "\"lastUpdate\"=>\"0\", \"stateMessage\"=>[{" \
 "\"lockType\"=>\"SHARED\", " \
 "\"tableName\"=>{\"namespace\"=>\"bnM0\", 
\"qualifier\"=>\"dGFibGU0\"}, " \
-"\"description\"=>\"description\"}]}\n" \
+"\"description\"=>\"description\"}], \"executed\"=>false}\n" \
 "1 row(s)\n\n",
 output)
 end

(hbase) branch branch-2.4 updated: HBASE-28212 Do not need to maintain rollback step when root procedure does not support rollback (#5538) (#5547)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.4 by this push:
 new b1bef8fcba8 HBASE-28212 Do not need to maintain rollback step when 
root procedure does not support rollback (#5538) (#5547)
b1bef8fcba8 is described below

commit b1bef8fcba8e471cc5f009c0508c2c5345e07b71
Author: Duo Zhang 
AuthorDate: Wed Nov 29 19:40:44 2023 +0800

HBASE-28212 Do not need to maintain rollback step when root procedure does 
not support rollback (#5538) (#5547)

Signed-off-by: GeorryHuang 
(cherry picked from commit 4b015e6a5486394d70bbf5fc0197e469c0987913)
---
 .../apache/hadoop/hbase/procedure2/Procedure.java  |  22 +-
 .../hadoop/hbase/procedure2/ProcedureExecutor.java | 346 ++---
 .../hadoop/hbase/procedure2/ProcedureUtil.java |   5 +
 .../hbase/procedure2/RootProcedureState.java   |  52 +++-
 .../hbase/procedure2/StateMachineProcedure.java|   5 +
 .../hbase/procedure2/ProcedureTestingUtility.java  |  37 ++-
 .../hbase/procedure2/TestProcedureRecovery.java|  15 +-
 .../procedure2/TestStateMachineProcedure.java  |   6 +
 .../hbase/procedure2/TestYieldProcedures.java  |   5 +
 .../store/wal/TestWALProcedureStore.java   |  10 +-
 .../src/main/protobuf/Procedure.proto  |   4 +
 .../hbase/master/assignment/TestRegionBypass.java  |  36 +--
 .../hbase/master/assignment/TestRollbackSCP.java   | 183 +++
 13 files changed, 573 insertions(+), 153 deletions(-)

diff --git 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
index a71026e6007..44639af1a9f 100644
--- 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
+++ 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
@@ -131,6 +131,9 @@ public abstract class Procedure implements 
Comparable implements 
Comparable implements 
Comparable {
 protected volatile boolean killAfterStoreUpdate = false;
 protected volatile boolean toggleKillAfterStoreUpdate = false;
 
+protected volatile boolean killBeforeStoreUpdateInRollback = false;
+protected volatile boolean toggleKillBeforeStoreUpdateInRollback = false;
+
 protected boolean shouldKillBeforeStoreUpdate() {
   final boolean kill = this.killBeforeStoreUpdate;
   if (this.toggleKillBeforeStoreUpdate) {
@@ -146,6 +151,16 @@ public class ProcedureExecutor {
 protected boolean shouldKillAfterStoreUpdate(final boolean isSuspended) {
   return (isSuspended && !killIfSuspended) ? false : 
shouldKillAfterStoreUpdate();
 }
+
+protected boolean shouldKillBeforeStoreUpdateInRollback() {
+  final boolean kill = this.killBeforeStoreUpdateInRollback;
+  if (this.toggleKillBeforeStoreUpdateInRollback) {
+this.killBeforeStoreUpdateInRollback = !kill;
+LOG.warn("Toggle KILL before store update in rollback to: "
+  + this.killBeforeStoreUpdateInRollback);
+  }
+  return kill;
+}
   }
 
   public interface ProcedureExecutorListener {
@@ -392,68 +407,10 @@ public class ProcedureExecutor {
 });
   }
 
-  private void loadProcedures(ProcedureIterator procIter) throws IOException {
-// 1. Build the rollback stack
-int runnableCount = 0;
-int failedCount = 0;
-int waitingCount = 0;
-int waitingTimeoutCount = 0;
-while (procIter.hasNext()) {
-  boolean finished = procIter.isNextFinished();
-  @SuppressWarnings("unchecked")
-  Procedure proc = procIter.next();
-  NonceKey nonceKey = proc.getNonceKey();
-  long procId = proc.getProcId();
-
-  if (finished) {
-completed.put(proc.getProcId(), new 
CompletedProcedureRetainer<>(proc));
-LOG.debug("Completed {}", proc);
-  } else {
-if (!proc.hasParent()) {
-  assert !proc.isFinished() : "unexpected finished procedure";
-  rollbackStack.put(proc.getProcId(), new RootProcedureState<>());
-}
-
-// add the procedure to the map
-proc.beforeReplay(getEnvironment());
-procedures.put(proc.getProcId(), proc);
-switch (proc.getState()) {
-  case RUNNABLE:
-runnableCount++;
-break;
-  case FAILED:
-failedCount++;
-break;
-  case WAITING:
-waitingCount++;
-break;
-  case WAITING_TIMEOUT:
-waitingTimeoutCount++;
-break;
-  default:
-break;
-}
-  }
-
-  if (nonceKey != null) {
-nonceKeysToProcIdsMap.put(nonceKey, procId); // add the nonce to the 
map
-  }
-}
-
-// 2. Initialize the stacks: In the old implement

(hbase) branch branch-2.5 updated: HBASE-28212 Do not need to maintain rollback step when root procedure does not support rollback (#5538) (#5547)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.5 by this push:
 new 8f70d84b809 HBASE-28212 Do not need to maintain rollback step when 
root procedure does not support rollback (#5538) (#5547)
8f70d84b809 is described below

commit 8f70d84b8095fd112bb141e10ba88cffdd54f5d1
Author: Duo Zhang 
AuthorDate: Wed Nov 29 19:40:44 2023 +0800

HBASE-28212 Do not need to maintain rollback step when root procedure does 
not support rollback (#5538) (#5547)

Signed-off-by: GeorryHuang 
(cherry picked from commit 4b015e6a5486394d70bbf5fc0197e469c0987913)
---
 .../apache/hadoop/hbase/procedure2/Procedure.java  |  22 +-
 .../hadoop/hbase/procedure2/ProcedureExecutor.java | 346 ++---
 .../hadoop/hbase/procedure2/ProcedureUtil.java |   5 +
 .../hbase/procedure2/RootProcedureState.java   |  52 +++-
 .../hbase/procedure2/StateMachineProcedure.java|   5 +
 .../hbase/procedure2/ProcedureTestingUtility.java  |  37 ++-
 .../hbase/procedure2/TestProcedureRecovery.java|  15 +-
 .../procedure2/TestStateMachineProcedure.java  |   6 +
 .../hbase/procedure2/TestYieldProcedures.java  |   5 +
 .../store/wal/TestWALProcedureStore.java   |  10 +-
 .../src/main/protobuf/Procedure.proto  |   4 +
 .../hbase/master/assignment/TestRegionBypass.java  |  36 +--
 .../hbase/master/assignment/TestRollbackSCP.java   | 186 +++
 13 files changed, 576 insertions(+), 153 deletions(-)

diff --git 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
index a71026e6007..44639af1a9f 100644
--- 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
+++ 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
@@ -131,6 +131,9 @@ public abstract class Procedure implements 
Comparable implements 
Comparable implements 
Comparable {
 protected volatile boolean killAfterStoreUpdate = false;
 protected volatile boolean toggleKillAfterStoreUpdate = false;
 
+protected volatile boolean killBeforeStoreUpdateInRollback = false;
+protected volatile boolean toggleKillBeforeStoreUpdateInRollback = false;
+
 protected boolean shouldKillBeforeStoreUpdate() {
   final boolean kill = this.killBeforeStoreUpdate;
   if (this.toggleKillBeforeStoreUpdate) {
@@ -148,6 +153,16 @@ public class ProcedureExecutor {
 protected boolean shouldKillAfterStoreUpdate(final boolean isSuspended) {
   return (isSuspended && !killIfSuspended) ? false : 
shouldKillAfterStoreUpdate();
 }
+
+protected boolean shouldKillBeforeStoreUpdateInRollback() {
+  final boolean kill = this.killBeforeStoreUpdateInRollback;
+  if (this.toggleKillBeforeStoreUpdateInRollback) {
+this.killBeforeStoreUpdateInRollback = !kill;
+LOG.warn("Toggle KILL before store update in rollback to: "
+  + this.killBeforeStoreUpdateInRollback);
+  }
+  return kill;
+}
   }
 
   public interface ProcedureExecutorListener {
@@ -394,68 +409,10 @@ public class ProcedureExecutor {
 });
   }
 
-  private void loadProcedures(ProcedureIterator procIter) throws IOException {
-// 1. Build the rollback stack
-int runnableCount = 0;
-int failedCount = 0;
-int waitingCount = 0;
-int waitingTimeoutCount = 0;
-while (procIter.hasNext()) {
-  boolean finished = procIter.isNextFinished();
-  @SuppressWarnings("unchecked")
-  Procedure proc = procIter.next();
-  NonceKey nonceKey = proc.getNonceKey();
-  long procId = proc.getProcId();
-
-  if (finished) {
-completed.put(proc.getProcId(), new 
CompletedProcedureRetainer<>(proc));
-LOG.debug("Completed {}", proc);
-  } else {
-if (!proc.hasParent()) {
-  assert !proc.isFinished() : "unexpected finished procedure";
-  rollbackStack.put(proc.getProcId(), new RootProcedureState<>());
-}
-
-// add the procedure to the map
-proc.beforeReplay(getEnvironment());
-procedures.put(proc.getProcId(), proc);
-switch (proc.getState()) {
-  case RUNNABLE:
-runnableCount++;
-break;
-  case FAILED:
-failedCount++;
-break;
-  case WAITING:
-waitingCount++;
-break;
-  case WAITING_TIMEOUT:
-waitingTimeoutCount++;
-break;
-  default:
-break;
-}
-  }
-
-  if (nonceKey != null) {
-nonceKeysToProcIdsMap.put(nonceKey, procId); // add the nonce to the 
map
-  }
-}
-
-// 2. Initialize the stacks: In the old implement

(hbase) branch branch-2 updated: HBASE-28212 Do not need to maintain rollback step when root procedure does not support rollback (#5538) (#5547)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2 by this push:
 new 0a8b620faff HBASE-28212 Do not need to maintain rollback step when 
root procedure does not support rollback (#5538) (#5547)
0a8b620faff is described below

commit 0a8b620faff1dcf90ba360ad777b6b7cd8da5a5b
Author: Duo Zhang 
AuthorDate: Wed Nov 29 19:40:44 2023 +0800

HBASE-28212 Do not need to maintain rollback step when root procedure does 
not support rollback (#5538) (#5547)

Signed-off-by: GeorryHuang 
(cherry picked from commit 4b015e6a5486394d70bbf5fc0197e469c0987913)
---
 .../apache/hadoop/hbase/procedure2/Procedure.java  |  22 +-
 .../hadoop/hbase/procedure2/ProcedureExecutor.java | 346 ++---
 .../hadoop/hbase/procedure2/ProcedureUtil.java |   5 +
 .../hbase/procedure2/RootProcedureState.java   |  52 +++-
 .../hbase/procedure2/StateMachineProcedure.java|   5 +
 .../hbase/procedure2/ProcedureTestingUtility.java  |  37 ++-
 .../hbase/procedure2/TestProcedureRecovery.java|  15 +-
 .../procedure2/TestStateMachineProcedure.java  |   6 +
 .../hbase/procedure2/TestYieldProcedures.java  |   5 +
 .../store/wal/TestWALProcedureStore.java   |  10 +-
 .../src/main/protobuf/Procedure.proto  |   4 +
 .../hbase/master/assignment/TestRegionBypass.java  |  36 +--
 .../hbase/master/assignment/TestRollbackSCP.java   | 186 +++
 13 files changed, 576 insertions(+), 153 deletions(-)

diff --git 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
index a71026e6007..44639af1a9f 100644
--- 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
+++ 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
@@ -131,6 +131,9 @@ public abstract class Procedure implements 
Comparable implements 
Comparable implements 
Comparable {
 protected volatile boolean killAfterStoreUpdate = false;
 protected volatile boolean toggleKillAfterStoreUpdate = false;
 
+protected volatile boolean killBeforeStoreUpdateInRollback = false;
+protected volatile boolean toggleKillBeforeStoreUpdateInRollback = false;
+
 protected boolean shouldKillBeforeStoreUpdate() {
   final boolean kill = this.killBeforeStoreUpdate;
   if (this.toggleKillBeforeStoreUpdate) {
@@ -148,6 +153,16 @@ public class ProcedureExecutor {
 protected boolean shouldKillAfterStoreUpdate(final boolean isSuspended) {
   return (isSuspended && !killIfSuspended) ? false : 
shouldKillAfterStoreUpdate();
 }
+
+protected boolean shouldKillBeforeStoreUpdateInRollback() {
+  final boolean kill = this.killBeforeStoreUpdateInRollback;
+  if (this.toggleKillBeforeStoreUpdateInRollback) {
+this.killBeforeStoreUpdateInRollback = !kill;
+LOG.warn("Toggle KILL before store update in rollback to: "
+  + this.killBeforeStoreUpdateInRollback);
+  }
+  return kill;
+}
   }
 
   public interface ProcedureExecutorListener {
@@ -394,68 +409,10 @@ public class ProcedureExecutor {
 });
   }
 
-  private void loadProcedures(ProcedureIterator procIter) throws IOException {
-// 1. Build the rollback stack
-int runnableCount = 0;
-int failedCount = 0;
-int waitingCount = 0;
-int waitingTimeoutCount = 0;
-while (procIter.hasNext()) {
-  boolean finished = procIter.isNextFinished();
-  @SuppressWarnings("unchecked")
-  Procedure proc = procIter.next();
-  NonceKey nonceKey = proc.getNonceKey();
-  long procId = proc.getProcId();
-
-  if (finished) {
-completed.put(proc.getProcId(), new 
CompletedProcedureRetainer<>(proc));
-LOG.debug("Completed {}", proc);
-  } else {
-if (!proc.hasParent()) {
-  assert !proc.isFinished() : "unexpected finished procedure";
-  rollbackStack.put(proc.getProcId(), new RootProcedureState<>());
-}
-
-// add the procedure to the map
-proc.beforeReplay(getEnvironment());
-procedures.put(proc.getProcId(), proc);
-switch (proc.getState()) {
-  case RUNNABLE:
-runnableCount++;
-break;
-  case FAILED:
-failedCount++;
-break;
-  case WAITING:
-waitingCount++;
-break;
-  case WAITING_TIMEOUT:
-waitingTimeoutCount++;
-break;
-  default:
-break;
-}
-  }
-
-  if (nonceKey != null) {
-nonceKeysToProcIdsMap.put(nonceKey, procId); // add the nonce to the 
map
-  }
-}
-
-// 2. Initialize the stacks: In the old implement

(hbase) branch branch-3 updated: HBASE-28212 Do not need to maintain rollback step when root procedure does not support rollback (#5538)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-3 by this push:
 new 7d6de8b948a HBASE-28212 Do not need to maintain rollback step when 
root procedure does not support rollback (#5538)
7d6de8b948a is described below

commit 7d6de8b948a088d316124d6587e6a540bc5b38f1
Author: Duo Zhang 
AuthorDate: Wed Nov 29 10:59:38 2023 +0800

HBASE-28212 Do not need to maintain rollback step when root procedure does 
not support rollback (#5538)

Signed-off-by: GeorryHuang 
(cherry picked from commit 4b015e6a5486394d70bbf5fc0197e469c0987913)
---
 .../apache/hadoop/hbase/procedure2/Procedure.java  |  22 +-
 .../hadoop/hbase/procedure2/ProcedureExecutor.java | 346 ++---
 .../hadoop/hbase/procedure2/ProcedureUtil.java |   5 +
 .../hbase/procedure2/RootProcedureState.java   |  52 +++-
 .../hbase/procedure2/StateMachineProcedure.java|   5 +
 .../hbase/procedure2/ProcedureTestingUtility.java  |  37 ++-
 .../hbase/procedure2/TestProcedureRecovery.java|  16 +-
 .../procedure2/TestStateMachineProcedure.java  |   6 +
 .../hbase/procedure2/TestYieldProcedures.java  |   6 +
 .../store/wal/TestWALProcedureStore.java   |  10 +-
 .../src/main/protobuf/server/Procedure.proto   |   4 +
 .../hbase/master/assignment/TestRegionBypass.java  |  36 +--
 .../hbase/master/assignment/TestRollbackSCP.java   | 186 +++
 13 files changed, 578 insertions(+), 153 deletions(-)

diff --git 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
index 43adba2bc21..7bd64fd9944 100644
--- 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
+++ 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
@@ -133,6 +133,9 @@ public abstract class Procedure implements 
Comparable implements 
Comparable implements 
Comparable {
 protected volatile boolean killAfterStoreUpdate = false;
 protected volatile boolean toggleKillAfterStoreUpdate = false;
 
+protected volatile boolean killBeforeStoreUpdateInRollback = false;
+protected volatile boolean toggleKillBeforeStoreUpdateInRollback = false;
+
 protected boolean shouldKillBeforeStoreUpdate() {
   final boolean kill = this.killBeforeStoreUpdate;
   if (this.toggleKillBeforeStoreUpdate) {
@@ -148,6 +153,16 @@ public class ProcedureExecutor {
 protected boolean shouldKillAfterStoreUpdate(final boolean isSuspended) {
   return (isSuspended && !killIfSuspended) ? false : 
shouldKillAfterStoreUpdate();
 }
+
+protected boolean shouldKillBeforeStoreUpdateInRollback() {
+  final boolean kill = this.killBeforeStoreUpdateInRollback;
+  if (this.toggleKillBeforeStoreUpdateInRollback) {
+this.killBeforeStoreUpdateInRollback = !kill;
+LOG.warn("Toggle KILL before store update in rollback to: "
+  + this.killBeforeStoreUpdateInRollback);
+  }
+  return kill;
+}
   }
 
   public interface ProcedureExecutorListener {
@@ -394,68 +409,10 @@ public class ProcedureExecutor {
 });
   }
 
-  private void loadProcedures(ProcedureIterator procIter) throws IOException {
-// 1. Build the rollback stack
-int runnableCount = 0;
-int failedCount = 0;
-int waitingCount = 0;
-int waitingTimeoutCount = 0;
-while (procIter.hasNext()) {
-  boolean finished = procIter.isNextFinished();
-  @SuppressWarnings("unchecked")
-  Procedure proc = procIter.next();
-  NonceKey nonceKey = proc.getNonceKey();
-  long procId = proc.getProcId();
-
-  if (finished) {
-completed.put(proc.getProcId(), new 
CompletedProcedureRetainer<>(proc));
-LOG.debug("Completed {}", proc);
-  } else {
-if (!proc.hasParent()) {
-  assert !proc.isFinished() : "unexpected finished procedure";
-  rollbackStack.put(proc.getProcId(), new RootProcedureState<>());
-}
-
-// add the procedure to the map
-proc.beforeReplay(getEnvironment());
-procedures.put(proc.getProcId(), proc);
-switch (proc.getState()) {
-  case RUNNABLE:
-runnableCount++;
-break;
-  case FAILED:
-failedCount++;
-break;
-  case WAITING:
-waitingCount++;
-break;
-  case WAITING_TIMEOUT:
-waitingTimeoutCount++;
-break;
-  default:
-break;
-}
-  }
-
-  if (nonceKey != null) {
-nonceKeysToProcIdsMap.put(nonceKey, procId); // add the nonce to the 
map
-  }
-}
-
-// 2. Initialize the stacks: In the old implementation, for procedures in 
FAILE

(hbase) branch master updated (4b015e6a548 -> 4d90b918a37)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


from 4b015e6a548 HBASE-28212 Do not need to maintain rollback step when 
root procedure does not support rollback (#5538)
 add 4d90b918a37 HBASE-28225 Bump cryptography in 
/dev-support/git-jira-release-audit (#5544)

No new revisions were added by this update.

Summary of changes:
 dev-support/git-jira-release-audit/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(hbase) branch master updated: HBASE-28212 Do not need to maintain rollback step when root procedure does not support rollback (#5538)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
 new 4b015e6a548 HBASE-28212 Do not need to maintain rollback step when 
root procedure does not support rollback (#5538)
4b015e6a548 is described below

commit 4b015e6a5486394d70bbf5fc0197e469c0987913
Author: Duo Zhang 
AuthorDate: Wed Nov 29 10:59:38 2023 +0800

HBASE-28212 Do not need to maintain rollback step when root procedure does 
not support rollback (#5538)

Signed-off-by: GeorryHuang 
---
 .../apache/hadoop/hbase/procedure2/Procedure.java  |  22 +-
 .../hadoop/hbase/procedure2/ProcedureExecutor.java | 346 ++---
 .../hadoop/hbase/procedure2/ProcedureUtil.java |   5 +
 .../hbase/procedure2/RootProcedureState.java   |  52 +++-
 .../hbase/procedure2/StateMachineProcedure.java|   5 +
 .../hbase/procedure2/ProcedureTestingUtility.java  |  37 ++-
 .../hbase/procedure2/TestProcedureRecovery.java|  16 +-
 .../procedure2/TestStateMachineProcedure.java  |   6 +
 .../hbase/procedure2/TestYieldProcedures.java  |   6 +
 .../store/wal/TestWALProcedureStore.java   |  10 +-
 .../src/main/protobuf/server/Procedure.proto   |   4 +
 .../hbase/master/assignment/TestRegionBypass.java  |  36 +--
 .../hbase/master/assignment/TestRollbackSCP.java   | 186 +++
 13 files changed, 578 insertions(+), 153 deletions(-)

diff --git 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
index 43adba2bc21..7bd64fd9944 100644
--- 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
+++ 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
@@ -133,6 +133,9 @@ public abstract class Procedure implements 
Comparable implements 
Comparable implements 
Comparable {
 protected volatile boolean killAfterStoreUpdate = false;
 protected volatile boolean toggleKillAfterStoreUpdate = false;
 
+protected volatile boolean killBeforeStoreUpdateInRollback = false;
+protected volatile boolean toggleKillBeforeStoreUpdateInRollback = false;
+
 protected boolean shouldKillBeforeStoreUpdate() {
   final boolean kill = this.killBeforeStoreUpdate;
   if (this.toggleKillBeforeStoreUpdate) {
@@ -148,6 +153,16 @@ public class ProcedureExecutor {
 protected boolean shouldKillAfterStoreUpdate(final boolean isSuspended) {
   return (isSuspended && !killIfSuspended) ? false : 
shouldKillAfterStoreUpdate();
 }
+
+protected boolean shouldKillBeforeStoreUpdateInRollback() {
+  final boolean kill = this.killBeforeStoreUpdateInRollback;
+  if (this.toggleKillBeforeStoreUpdateInRollback) {
+this.killBeforeStoreUpdateInRollback = !kill;
+LOG.warn("Toggle KILL before store update in rollback to: "
+  + this.killBeforeStoreUpdateInRollback);
+  }
+  return kill;
+}
   }
 
   public interface ProcedureExecutorListener {
@@ -394,68 +409,10 @@ public class ProcedureExecutor {
 });
   }
 
-  private void loadProcedures(ProcedureIterator procIter) throws IOException {
-// 1. Build the rollback stack
-int runnableCount = 0;
-int failedCount = 0;
-int waitingCount = 0;
-int waitingTimeoutCount = 0;
-while (procIter.hasNext()) {
-  boolean finished = procIter.isNextFinished();
-  @SuppressWarnings("unchecked")
-  Procedure proc = procIter.next();
-  NonceKey nonceKey = proc.getNonceKey();
-  long procId = proc.getProcId();
-
-  if (finished) {
-completed.put(proc.getProcId(), new 
CompletedProcedureRetainer<>(proc));
-LOG.debug("Completed {}", proc);
-  } else {
-if (!proc.hasParent()) {
-  assert !proc.isFinished() : "unexpected finished procedure";
-  rollbackStack.put(proc.getProcId(), new RootProcedureState<>());
-}
-
-// add the procedure to the map
-proc.beforeReplay(getEnvironment());
-procedures.put(proc.getProcId(), proc);
-switch (proc.getState()) {
-  case RUNNABLE:
-runnableCount++;
-break;
-  case FAILED:
-failedCount++;
-break;
-  case WAITING:
-waitingCount++;
-break;
-  case WAITING_TIMEOUT:
-waitingTimeoutCount++;
-break;
-  default:
-break;
-}
-  }
-
-  if (nonceKey != null) {
-nonceKeysToProcIdsMap.put(nonceKey, procId); // add the nonce to the 
map
-  }
-}
-
-// 2. Initialize the stacks: In the old implementation, for procedures in 
FAILED state, we will
-// push it into the ProcedureScheduler directly to ex

(hbase) branch master updated: HBASE-28031 TestClusterScopeQuotaThrottle is still failing with broken WAL writer (#5539)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
 new 6f8b288c425 HBASE-28031 TestClusterScopeQuotaThrottle is still failing 
with broken WAL writer (#5539)
6f8b288c425 is described below

commit 6f8b288c4254e407a2969e8cfd06468144150259
Author: Duo Zhang 
AuthorDate: Tue Nov 28 18:28:34 2023 +0800

HBASE-28031 TestClusterScopeQuotaThrottle is still failing with broken WAL 
writer (#5539)

Limit the scope for EnvironmentEdge injection

Signed-off-by: Guanghao Zhang 
---
 .../util/EnvironmentEdgeManagerTestHelper.java | 32 ++
 .../quotas/TestClusterScopeQuotaThrottle.java  |  3 --
 .../hadoop/hbase/quotas/ThrottleQuotaTestUtil.java |  4 ++-
 3 files changed, 35 insertions(+), 4 deletions(-)

diff --git 
a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java
 
b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java
index 684247248dc..73e7f1623ef 100644
--- 
a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java
+++ 
b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java
@@ -33,4 +33,36 @@ public final class EnvironmentEdgeManagerTestHelper {
   public static void injectEdge(EnvironmentEdge edge) {
 EnvironmentEdgeManager.injectEdge(edge);
   }
+
+  private static final class PackageEnvironmentEdgeWrapper implements 
EnvironmentEdge {
+
+private final EnvironmentEdge delegate;
+
+private final String packageName;
+
+PackageEnvironmentEdgeWrapper(EnvironmentEdge delegate, String 
packageName) {
+  this.delegate = delegate;
+  this.packageName = packageName;
+}
+
+@Override
+public long currentTime() {
+  StackTraceElement[] elements = new Exception().getStackTrace();
+  // the first element is us, the second one is EnvironmentEdgeManager, so 
let's check the third
+  // one
+  if (elements.length > 2 && 
elements[2].getClassName().startsWith(packageName)) {
+return delegate.currentTime();
+  } else {
+return System.currentTimeMillis();
+  }
+}
+  }
+
+  /**
+   * Inject a {@link EnvironmentEdge} which only takes effect when calling 
directly from the classes
+   * in the given package.
+   */
+  public static void injectEdgeForPackage(EnvironmentEdge edge, String 
packageName) {
+injectEdge(new PackageEnvironmentEdgeWrapper(edge, packageName));
+  }
 }
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestClusterScopeQuotaThrottle.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestClusterScopeQuotaThrottle.java
index c617c34800f..b34f722e2e7 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestClusterScopeQuotaThrottle.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestClusterScopeQuotaThrottle.java
@@ -75,9 +75,6 @@ public class TestClusterScopeQuotaThrottle {
 TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
 TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
 
TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", 
true);
-// disable stream slow monitor check, as in this test we inject our own 
EnvironmentEdge
-
TEST_UTIL.getConfiguration().setInt("hbase.regionserver.async.wal.min.slow.detect.count",
-  Integer.MAX_VALUE);
 TEST_UTIL.startMiniCluster(2);
 TEST_UTIL.waitTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME);
 QuotaCache.TEST_FORCE_REFRESH = true;
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
index 93eae8dfccf..a6e93b663c0 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
@@ -42,7 +42,9 @@ public final class ThrottleQuotaTestUtil {
   private final static int REFRESH_TIME = 30 * 6;
   static {
 envEdge.setValue(EnvironmentEdgeManager.currentTime());
-EnvironmentEdgeManagerTestHelper.injectEdge(envEdge);
+// only active the envEdge for quotas package
+EnvironmentEdgeManagerTestHelper.injectEdgeForPackage(envEdge,
+  ThrottleQuotaTestUtil.class.getPackage().getName());
   }
 
   private ThrottleQuotaTestUtil() {

(hbase) branch branch-3 updated: HBASE-28218 Add a check for getQueueStorage().hasData() in the getDeletableFiles method of ReplicationLogCleaner (#5536)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-3 by this push:
 new d5c14b5e059 HBASE-28218 Add a check for getQueueStorage().hasData() in 
the getDeletableFiles method of ReplicationLogCleaner (#5536)
d5c14b5e059 is described below

commit d5c14b5e05920fa1fa568b5c0c6cb76e7e4c3706
Author: hiping-tech <58875741+hiping-t...@users.noreply.github.com>
AuthorDate: Fri Nov 24 10:29:34 2023 +0800

HBASE-28218 Add a check for getQueueStorage().hasData() in the 
getDeletableFiles method of ReplicationLogCleaner (#5536)

Co-authored-by: lvhaiping.lhp 
Signed-off-by: Duo Zhang 
(cherry picked from commit dba900f778614b392e5e83522cdda72487558f48)
---
 .../hadoop/hbase/replication/master/ReplicationLogCleaner.java| 8 
 1 file changed, 8 insertions(+)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
index 6ebcac7e453..7fc8feae72a 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
@@ -192,6 +192,14 @@ public class ReplicationLogCleaner extends 
BaseLogCleanerDelegate {
 if (this.getConf() == null) {
   return files;
 }
+try {
+  if (!rpm.getQueueStorage().hasData()) {
+return files;
+  }
+} catch (ReplicationException e) {
+  LOG.error("Error occurred while executing queueStorage.hasData()", e);
+  return Collections.emptyList();
+}
 if (!canFilter) {
   // We can not delete anything if there are AddPeerProcedure running at 
the same time
   // See HBASE-27214 for more details.

(hbase) branch master updated: HBASE-28218 Add a check for getQueueStorage().hasData() in the getDeletableFiles method of ReplicationLogCleaner (#5536)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
 new dba900f7786 HBASE-28218 Add a check for getQueueStorage().hasData() in 
the getDeletableFiles method of ReplicationLogCleaner (#5536)
dba900f7786 is described below

commit dba900f778614b392e5e83522cdda72487558f48
Author: hiping-tech <58875741+hiping-t...@users.noreply.github.com>
AuthorDate: Fri Nov 24 10:29:34 2023 +0800

HBASE-28218 Add a check for getQueueStorage().hasData() in the 
getDeletableFiles method of ReplicationLogCleaner (#5536)

Co-authored-by: lvhaiping.lhp 
Signed-off-by: Duo Zhang 
---
 .../hadoop/hbase/replication/master/ReplicationLogCleaner.java| 8 
 1 file changed, 8 insertions(+)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
index 6ebcac7e453..7fc8feae72a 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
@@ -192,6 +192,14 @@ public class ReplicationLogCleaner extends 
BaseLogCleanerDelegate {
 if (this.getConf() == null) {
   return files;
 }
+try {
+  if (!rpm.getQueueStorage().hasData()) {
+return files;
+  }
+} catch (ReplicationException e) {
+  LOG.error("Error occurred while executing queueStorage.hasData()", e);
+  return Collections.emptyList();
+}
 if (!canFilter) {
   // We can not delete anything if there are AddPeerProcedure running at 
the same time
   // See HBASE-27214 for more details.

(hbase) branch branch-2.5 updated: HBASE-28210 Addendum fix TestProcedureAdmin (#5532)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.5 by this push:
 new ffb91fc143a HBASE-28210 Addendum fix TestProcedureAdmin (#5532)
ffb91fc143a is described below

commit ffb91fc143aff63e78b654769fea4843dc7ad8f4
Author: Duo Zhang 
AuthorDate: Wed Nov 22 10:37:42 2023 +0800

HBASE-28210 Addendum fix TestProcedureAdmin (#5532)

(cherry picked from commit b1ccf33382780c27d96340dcfd1aec312ed2cb73)
---
 .../apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java  | 8 ++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java
index e84daca0732..6a75c878a71 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java
@@ -155,8 +155,7 @@ public class TestProcedureAdmin {
 final TableName tableName = TableName.valueOf(name.getMethodName());
 final ProcedureExecutor procExec = 
getMasterProcedureExecutor();
 
-RegionInfo[] regions =
-  MasterProcedureTestingUtility.createTable(procExec, tableName, null, 
"f");
+MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f");
 ProcedureTestingUtility.waitNoProcedureRunning(procExec);
 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
 // Submit a procedure
@@ -164,6 +163,11 @@ public class TestProcedureAdmin {
   .submitProcedure(new DisableTableProcedure(procExec.getEnvironment(), 
tableName, true));
 // Wait for one step to complete
 ProcedureTestingUtility.waitProcedure(procExec, procId);
+// After HBASE-28210, the injection of kill before update is moved before 
we add rollback
+// step, so here we need to run two steps, otherwise we will not consider 
the procedure as
+// executed
+MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec);
+ProcedureTestingUtility.waitProcedure(procExec, procId);
 
 // Set the mayInterruptIfRunning flag to false
 boolean abortResult = procExec.abort(procId, false);

(hbase) branch branch-3 updated: HBASE-28210 Addendum fix TestProcedureAdmin (#5532)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-3 by this push:
 new 92f8066b3f0 HBASE-28210 Addendum fix TestProcedureAdmin (#5532)
92f8066b3f0 is described below

commit 92f8066b3f05f0a9f2c64a2cd8bde78eeaf1f388
Author: Duo Zhang 
AuthorDate: Wed Nov 22 10:37:42 2023 +0800

HBASE-28210 Addendum fix TestProcedureAdmin (#5532)

(cherry picked from commit b1ccf33382780c27d96340dcfd1aec312ed2cb73)
---
 .../apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java  | 8 ++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java
index 452a73d26ee..94539572a99 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java
@@ -155,8 +155,7 @@ public class TestProcedureAdmin {
 final TableName tableName = TableName.valueOf(name.getMethodName());
 final ProcedureExecutor procExec = 
getMasterProcedureExecutor();
 
-RegionInfo[] regions =
-  MasterProcedureTestingUtility.createTable(procExec, tableName, null, 
"f");
+MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f");
 ProcedureTestingUtility.waitNoProcedureRunning(procExec);
 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
 // Submit a procedure
@@ -164,6 +163,11 @@ public class TestProcedureAdmin {
   .submitProcedure(new DisableTableProcedure(procExec.getEnvironment(), 
tableName, true));
 // Wait for one step to complete
 ProcedureTestingUtility.waitProcedure(procExec, procId);
+// After HBASE-28210, the injection of kill before update is moved before 
we add rollback
+// step, so here we need to run two steps, otherwise we will not consider 
the procedure as
+// executed
+MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec);
+ProcedureTestingUtility.waitProcedure(procExec, procId);
 
 // Set the mayInterruptIfRunning flag to false
 boolean abortResult = procExec.abort(procId, false);

(hbase) branch branch-2 updated: HBASE-28210 Addendum fix TestProcedureAdmin (#5532)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2 by this push:
 new 76e90023e2c HBASE-28210 Addendum fix TestProcedureAdmin (#5532)
76e90023e2c is described below

commit 76e90023e2cffc76aedc5e172f85c8cb47235e48
Author: Duo Zhang 
AuthorDate: Wed Nov 22 10:37:42 2023 +0800

HBASE-28210 Addendum fix TestProcedureAdmin (#5532)

(cherry picked from commit b1ccf33382780c27d96340dcfd1aec312ed2cb73)
---
 .../apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java  | 8 ++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java
index e84daca0732..6a75c878a71 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java
@@ -155,8 +155,7 @@ public class TestProcedureAdmin {
 final TableName tableName = TableName.valueOf(name.getMethodName());
 final ProcedureExecutor procExec = 
getMasterProcedureExecutor();
 
-RegionInfo[] regions =
-  MasterProcedureTestingUtility.createTable(procExec, tableName, null, 
"f");
+MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f");
 ProcedureTestingUtility.waitNoProcedureRunning(procExec);
 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
 // Submit a procedure
@@ -164,6 +163,11 @@ public class TestProcedureAdmin {
   .submitProcedure(new DisableTableProcedure(procExec.getEnvironment(), 
tableName, true));
 // Wait for one step to complete
 ProcedureTestingUtility.waitProcedure(procExec, procId);
+// After HBASE-28210, the injection of kill before update is moved before 
we add rollback
+// step, so here we need to run two steps, otherwise we will not consider 
the procedure as
+// executed
+MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec);
+ProcedureTestingUtility.waitProcedure(procExec, procId);
 
 // Set the mayInterruptIfRunning flag to false
 boolean abortResult = procExec.abort(procId, false);

(hbase) branch branch-2.4 updated: HBASE-28210 Addendum fix TestProcedureAdmin (#5532)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.4 by this push:
 new 35efd2b8050 HBASE-28210 Addendum fix TestProcedureAdmin (#5532)
35efd2b8050 is described below

commit 35efd2b805039fb73d7be9df1894f0f0e125216e
Author: Duo Zhang 
AuthorDate: Wed Nov 22 10:37:42 2023 +0800

HBASE-28210 Addendum fix TestProcedureAdmin (#5532)

(cherry picked from commit b1ccf33382780c27d96340dcfd1aec312ed2cb73)
---
 .../apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java  | 8 ++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java
index e84daca0732..6a75c878a71 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java
@@ -155,8 +155,7 @@ public class TestProcedureAdmin {
 final TableName tableName = TableName.valueOf(name.getMethodName());
 final ProcedureExecutor procExec = 
getMasterProcedureExecutor();
 
-RegionInfo[] regions =
-  MasterProcedureTestingUtility.createTable(procExec, tableName, null, 
"f");
+MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f");
 ProcedureTestingUtility.waitNoProcedureRunning(procExec);
 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
 // Submit a procedure
@@ -164,6 +163,11 @@ public class TestProcedureAdmin {
   .submitProcedure(new DisableTableProcedure(procExec.getEnvironment(), 
tableName, true));
 // Wait for one step to complete
 ProcedureTestingUtility.waitProcedure(procExec, procId);
+// After HBASE-28210, the injection of kill before update is moved before 
we add rollback
+// step, so here we need to run two steps, otherwise we will not consider 
the procedure as
+// executed
+MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec);
+ProcedureTestingUtility.waitProcedure(procExec, procId);
 
 // Set the mayInterruptIfRunning flag to false
 boolean abortResult = procExec.abort(procId, false);

(hbase) branch master updated: HBASE-28210 Addendum fix TestProcedureAdmin (#5532)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
 new b1ccf333827 HBASE-28210 Addendum fix TestProcedureAdmin (#5532)
b1ccf333827 is described below

commit b1ccf33382780c27d96340dcfd1aec312ed2cb73
Author: Duo Zhang 
AuthorDate: Wed Nov 22 10:37:42 2023 +0800

HBASE-28210 Addendum fix TestProcedureAdmin (#5532)
---
 .../apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java  | 8 ++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java
index 452a73d26ee..94539572a99 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java
@@ -155,8 +155,7 @@ public class TestProcedureAdmin {
 final TableName tableName = TableName.valueOf(name.getMethodName());
 final ProcedureExecutor procExec = 
getMasterProcedureExecutor();
 
-RegionInfo[] regions =
-  MasterProcedureTestingUtility.createTable(procExec, tableName, null, 
"f");
+MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f");
 ProcedureTestingUtility.waitNoProcedureRunning(procExec);
 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
 // Submit a procedure
@@ -164,6 +163,11 @@ public class TestProcedureAdmin {
   .submitProcedure(new DisableTableProcedure(procExec.getEnvironment(), 
tableName, true));
 // Wait for one step to complete
 ProcedureTestingUtility.waitProcedure(procExec, procId);
+// After HBASE-28210, the injection of kill before update is moved before 
we add rollback
+// step, so here we need to run two steps, otherwise we will not consider 
the procedure as
+// executed
+MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec);
+ProcedureTestingUtility.waitProcedure(procExec, procId);
 
 // Set the mayInterruptIfRunning flag to false
 boolean abortResult = procExec.abort(procId, false);

(hbase) branch branch-3 updated: HBASE-28210 There could be holes in stack ids when loading procedures (#5531)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-3 by this push:
 new cb2f53dfbb0 HBASE-28210 There could be holes in stack ids when loading 
procedures (#5531)
cb2f53dfbb0 is described below

commit cb2f53dfbb0748491c1751acb14cdf65915c6020
Author: Duo Zhang 
AuthorDate: Tue Nov 21 14:29:13 2023 +0800

HBASE-28210 There could be holes in stack ids when loading procedures 
(#5531)

Signed-off-by: Wellington Chevreuil 
(cherry picked from commit e88daed9fbf2993fdc7feb2ba0127baa451a92cd)
---
 .../hadoop/hbase/procedure2/ProcedureExecutor.java |  10 +-
 .../hadoop/hbase/procedure2/TestStackIdHoles.java  | 228 +
 2 files changed, 234 insertions(+), 4 deletions(-)

diff --git 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
index 696dbb71b9f..46ce065b877 100644
--- 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
+++ 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
@@ -1695,9 +1695,6 @@ public class ProcedureExecutor {
 }
   }
 
-  // Add the procedure to the stack
-  procStack.addRollbackStep(procedure);
-
   // allows to kill the executor before something is stored to the wal.
   // useful to test the procedure recovery.
   if (
@@ -1715,7 +1712,12 @@ public class ProcedureExecutor {
   // Commit the transaction even if a suspend (state may have changed). 
Note this append
   // can take a bunch of time to complete.
   if (procedure.needPersistence()) {
-updateStoreOnExec(procStack, procedure, subprocs);
+// Add the procedure to the stack
+// See HBASE-28210 on why we need synchronized here
+synchronized (procStack) {
+  procStack.addRollbackStep(procedure);
+  updateStoreOnExec(procStack, procedure, subprocs);
+}
   }
 
   // if the store is not running we are aborting
diff --git 
a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestStackIdHoles.java
 
b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestStackIdHoles.java
new file mode 100644
index 000..4708df44745
--- /dev/null
+++ 
b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestStackIdHoles.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure2;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.LinkedHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseCommonTestingUtil;
+import 
org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.NoopProcedure;
+import org.apache.hadoop.hbase.procedure2.store.ProcedureStoreBase;
+import org.apache.hadoop.hbase.procedure2.store.ProcedureTree;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.AtomicUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
+
+/**
+ * Testcase for HBASE-28210, where we persist the procedure which has been 
inserted later to
+ * {@link RootProcedureState} first and then crash, and then cause holes in 
stack ids when loading,
+ * and finally fail the start up of master.
+ */
+@Category({ MasterTests.class, SmallTests.class })
+public class TestStackIdHoles {
+
+  @ClassRule
+  public static final HBaseClassTestRule CLASS_RULE =
+HBaseClassTestRule.forClass(TestStackIdHoles.class);
+
+  private final class DummyProcedureStore extends ProcedureStoreBase {
+
+pr

(hbase) branch branch-2 updated: HBASE-28210 There could be holes in stack ids when loading procedures (#5531)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2 by this push:
 new 79c293bb7f1 HBASE-28210 There could be holes in stack ids when loading 
procedures (#5531)
79c293bb7f1 is described below

commit 79c293bb7f1e30a2cd525db9bbfb69e1f263ab31
Author: Duo Zhang 
AuthorDate: Tue Nov 21 14:29:13 2023 +0800

HBASE-28210 There could be holes in stack ids when loading procedures 
(#5531)

Signed-off-by: Wellington Chevreuil 
(cherry picked from commit e88daed9fbf2993fdc7feb2ba0127baa451a92cd)
---
 .../hadoop/hbase/procedure2/ProcedureExecutor.java |  10 +-
 .../hadoop/hbase/procedure2/TestStackIdHoles.java  | 228 +
 2 files changed, 234 insertions(+), 4 deletions(-)

diff --git 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
index 696dbb71b9f..46ce065b877 100644
--- 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
+++ 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
@@ -1695,9 +1695,6 @@ public class ProcedureExecutor {
 }
   }
 
-  // Add the procedure to the stack
-  procStack.addRollbackStep(procedure);
-
   // allows to kill the executor before something is stored to the wal.
   // useful to test the procedure recovery.
   if (
@@ -1715,7 +1712,12 @@ public class ProcedureExecutor {
   // Commit the transaction even if a suspend (state may have changed). 
Note this append
   // can take a bunch of time to complete.
   if (procedure.needPersistence()) {
-updateStoreOnExec(procStack, procedure, subprocs);
+// Add the procedure to the stack
+// See HBASE-28210 on why we need synchronized here
+synchronized (procStack) {
+  procStack.addRollbackStep(procedure);
+  updateStoreOnExec(procStack, procedure, subprocs);
+}
   }
 
   // if the store is not running we are aborting
diff --git 
a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestStackIdHoles.java
 
b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestStackIdHoles.java
new file mode 100644
index 000..b4addae22d8
--- /dev/null
+++ 
b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestStackIdHoles.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure2;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.LinkedHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
+import 
org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.NoopProcedure;
+import org.apache.hadoop.hbase.procedure2.store.ProcedureStoreBase;
+import org.apache.hadoop.hbase.procedure2.store.ProcedureTree;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.AtomicUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
+
+/**
+ * Testcase for HBASE-28210, where we persist the procedure which has been 
inserted later to
+ * {@link RootProcedureState} first and then crash, and then cause holes in 
stack ids when loading,
+ * and finally fail the start up of master.
+ */
+@Category({ MasterTests.class, SmallTests.class })
+public class TestStackIdHoles {
+
+  @ClassRule
+  public static final HBaseClassTestRule CLASS_RULE =
+HBaseClassTestRule.forClass(TestStackIdHoles.class);
+
+  private final class DummyProcedureStore extends ProcedureStoreBase {
+
+pr

(hbase) branch branch-2.5 updated: HBASE-28210 There could be holes in stack ids when loading procedures (#5531)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.5 by this push:
 new ac529e4f3a6 HBASE-28210 There could be holes in stack ids when loading 
procedures (#5531)
ac529e4f3a6 is described below

commit ac529e4f3a641d3d957cf9060b6db571c0f0c3e4
Author: Duo Zhang 
AuthorDate: Tue Nov 21 14:29:13 2023 +0800

HBASE-28210 There could be holes in stack ids when loading procedures 
(#5531)

Signed-off-by: Wellington Chevreuil 
(cherry picked from commit e88daed9fbf2993fdc7feb2ba0127baa451a92cd)
---
 .../hadoop/hbase/procedure2/ProcedureExecutor.java |  10 +-
 .../hadoop/hbase/procedure2/TestStackIdHoles.java  | 228 +
 2 files changed, 234 insertions(+), 4 deletions(-)

diff --git 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
index 696dbb71b9f..46ce065b877 100644
--- 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
+++ 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
@@ -1695,9 +1695,6 @@ public class ProcedureExecutor {
 }
   }
 
-  // Add the procedure to the stack
-  procStack.addRollbackStep(procedure);
-
   // allows to kill the executor before something is stored to the wal.
   // useful to test the procedure recovery.
   if (
@@ -1715,7 +1712,12 @@ public class ProcedureExecutor {
   // Commit the transaction even if a suspend (state may have changed). 
Note this append
   // can take a bunch of time to complete.
   if (procedure.needPersistence()) {
-updateStoreOnExec(procStack, procedure, subprocs);
+// Add the procedure to the stack
+// See HBASE-28210 on why we need synchronized here
+synchronized (procStack) {
+  procStack.addRollbackStep(procedure);
+  updateStoreOnExec(procStack, procedure, subprocs);
+}
   }
 
   // if the store is not running we are aborting
diff --git 
a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestStackIdHoles.java
 
b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestStackIdHoles.java
new file mode 100644
index 000..b4addae22d8
--- /dev/null
+++ 
b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestStackIdHoles.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure2;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.LinkedHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
+import 
org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.NoopProcedure;
+import org.apache.hadoop.hbase.procedure2.store.ProcedureStoreBase;
+import org.apache.hadoop.hbase.procedure2.store.ProcedureTree;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.AtomicUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
+
+/**
+ * Testcase for HBASE-28210, where we persist the procedure which has been 
inserted later to
+ * {@link RootProcedureState} first and then crash, and then cause holes in 
stack ids when loading,
+ * and finally fail the start up of master.
+ */
+@Category({ MasterTests.class, SmallTests.class })
+public class TestStackIdHoles {
+
+  @ClassRule
+  public static final HBaseClassTestRule CLASS_RULE =
+HBaseClassTestRule.forClass(TestStackIdHoles.class);
+
+  private final class DummyProcedureStore extends ProcedureStoreBase {
+
+pr

(hbase) branch branch-2.4 updated: HBASE-28210 There could be holes in stack ids when loading procedures (#5531)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.4 by this push:
 new 6ae982e6c40 HBASE-28210 There could be holes in stack ids when loading 
procedures (#5531)
6ae982e6c40 is described below

commit 6ae982e6c40ff004ea50ef786307eec30716e162
Author: Duo Zhang 
AuthorDate: Tue Nov 21 14:29:13 2023 +0800

HBASE-28210 There could be holes in stack ids when loading procedures 
(#5531)

Signed-off-by: Wellington Chevreuil 
(cherry picked from commit e88daed9fbf2993fdc7feb2ba0127baa451a92cd)
---
 .../hadoop/hbase/procedure2/ProcedureExecutor.java |  10 +-
 .../hadoop/hbase/procedure2/TestStackIdHoles.java  | 228 +
 2 files changed, 234 insertions(+), 4 deletions(-)

diff --git 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
index 279809958d7..5d436122f9f 100644
--- 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
+++ 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
@@ -1691,9 +1691,6 @@ public class ProcedureExecutor {
 }
   }
 
-  // Add the procedure to the stack
-  procStack.addRollbackStep(procedure);
-
   // allows to kill the executor before something is stored to the wal.
   // useful to test the procedure recovery.
   if (
@@ -1711,7 +1708,12 @@ public class ProcedureExecutor {
   // Commit the transaction even if a suspend (state may have changed). 
Note this append
   // can take a bunch of time to complete.
   if (procedure.needPersistence()) {
-updateStoreOnExec(procStack, procedure, subprocs);
+// Add the procedure to the stack
+// See HBASE-28210 on why we need synchronized here
+synchronized (procStack) {
+  procStack.addRollbackStep(procedure);
+  updateStoreOnExec(procStack, procedure, subprocs);
+}
   }
 
   // if the store is not running we are aborting
diff --git 
a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestStackIdHoles.java
 
b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestStackIdHoles.java
new file mode 100644
index 000..b4addae22d8
--- /dev/null
+++ 
b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestStackIdHoles.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure2;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.LinkedHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
+import 
org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.NoopProcedure;
+import org.apache.hadoop.hbase.procedure2.store.ProcedureStoreBase;
+import org.apache.hadoop.hbase.procedure2.store.ProcedureTree;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.AtomicUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
+
+/**
+ * Testcase for HBASE-28210, where we persist the procedure which has been 
inserted later to
+ * {@link RootProcedureState} first and then crash, and then cause holes in 
stack ids when loading,
+ * and finally fail the start up of master.
+ */
+@Category({ MasterTests.class, SmallTests.class })
+public class TestStackIdHoles {
+
+  @ClassRule
+  public static final HBaseClassTestRule CLASS_RULE =
+HBaseClassTestRule.forClass(TestStackIdHoles.class);
+
+  private final class DummyProcedureStore extends ProcedureStoreBase {
+
+pr

(hbase) branch master updated: HBASE-28210 There could be holes in stack ids when loading procedures (#5531)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
 new e88daed9fbf HBASE-28210 There could be holes in stack ids when loading 
procedures (#5531)
e88daed9fbf is described below

commit e88daed9fbf2993fdc7feb2ba0127baa451a92cd
Author: Duo Zhang 
AuthorDate: Tue Nov 21 14:29:13 2023 +0800

HBASE-28210 There could be holes in stack ids when loading procedures 
(#5531)

Signed-off-by: Wellington Chevreuil 
---
 .../hadoop/hbase/procedure2/ProcedureExecutor.java |  10 +-
 .../hadoop/hbase/procedure2/TestStackIdHoles.java  | 228 +
 2 files changed, 234 insertions(+), 4 deletions(-)

diff --git 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
index 696dbb71b9f..46ce065b877 100644
--- 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
+++ 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
@@ -1695,9 +1695,6 @@ public class ProcedureExecutor {
 }
   }
 
-  // Add the procedure to the stack
-  procStack.addRollbackStep(procedure);
-
   // allows to kill the executor before something is stored to the wal.
   // useful to test the procedure recovery.
   if (
@@ -1715,7 +1712,12 @@ public class ProcedureExecutor {
   // Commit the transaction even if a suspend (state may have changed). 
Note this append
   // can take a bunch of time to complete.
   if (procedure.needPersistence()) {
-updateStoreOnExec(procStack, procedure, subprocs);
+// Add the procedure to the stack
+// See HBASE-28210 on why we need synchronized here
+synchronized (procStack) {
+  procStack.addRollbackStep(procedure);
+  updateStoreOnExec(procStack, procedure, subprocs);
+}
   }
 
   // if the store is not running we are aborting
diff --git 
a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestStackIdHoles.java
 
b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestStackIdHoles.java
new file mode 100644
index 000..4708df44745
--- /dev/null
+++ 
b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestStackIdHoles.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure2;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.LinkedHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseCommonTestingUtil;
+import 
org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.NoopProcedure;
+import org.apache.hadoop.hbase.procedure2.store.ProcedureStoreBase;
+import org.apache.hadoop.hbase.procedure2.store.ProcedureTree;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.AtomicUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
+
+/**
+ * Testcase for HBASE-28210, where we persist the procedure which has been 
inserted later to
+ * {@link RootProcedureState} first and then crash, and then cause holes in 
stack ids when loading,
+ * and finally fail the start up of master.
+ */
+@Category({ MasterTests.class, SmallTests.class })
+public class TestStackIdHoles {
+
+  @ClassRule
+  public static final HBaseClassTestRule CLASS_RULE =
+HBaseClassTestRule.forClass(TestStackIdHoles.class);
+
+  private final class DummyProcedureStore extends ProcedureStoreBase {
+
+private int numThreads;
+
+private final LinkedHashMap procMap =
+   

(hbase) branch master updated: HBASE-28031 TestClusterScopeQuotaThrottle is still failing with broken WAL writer (#5529)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
 new 9631af791e0 HBASE-28031 TestClusterScopeQuotaThrottle is still failing 
with broken WAL writer (#5529)
9631af791e0 is described below

commit 9631af791e0df5744c6df0c90993e033d8a8aef4
Author: Duo Zhang 
AuthorDate: Mon Nov 20 10:41:35 2023 +0800

HBASE-28031 TestClusterScopeQuotaThrottle is still failing with broken WAL 
writer (#5529)

Signed-off-by: Guanghao Zhang 
---
 .../org/apache/hadoop/hbase/quotas/TestClusterScopeQuotaThrottle.java  | 3 +++
 1 file changed, 3 insertions(+)

diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestClusterScopeQuotaThrottle.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestClusterScopeQuotaThrottle.java
index b34f722e2e7..c617c34800f 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestClusterScopeQuotaThrottle.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestClusterScopeQuotaThrottle.java
@@ -75,6 +75,9 @@ public class TestClusterScopeQuotaThrottle {
 TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
 TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
 
TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", 
true);
+// disable stream slow monitor check, as in this test we inject our own 
EnvironmentEdge
+
TEST_UTIL.getConfiguration().setInt("hbase.regionserver.async.wal.min.slow.detect.count",
+  Integer.MAX_VALUE);
 TEST_UTIL.startMiniCluster(2);
 TEST_UTIL.waitTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME);
 QuotaCache.TEST_FORCE_REFRESH = true;

(hbase) branch branch-2 updated: HBASE-28191 Meta browser can happen NPE when server or targetServer of region is null (#5510)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2 by this push:
 new abfe38169ce HBASE-28191 Meta browser can happen NPE when server or 
targetServer of region is null (#5510)
abfe38169ce is described below

commit abfe38169ce63df04d213afe46475fb1fefa3d40
Author: guluo 
AuthorDate: Fri Nov 10 22:26:19 2023 +0800

HBASE-28191 Meta browser can happen NPE when server or targetServer of 
region is null (#5510)

Signed-off-by: Duo Zhang 
---
 hbase-server/src/main/resources/hbase-webapps/master/table.jsp | 7 ---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp 
b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
index 2e3c32b298c..567e3c05eda 100644
--- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
+++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
@@ -517,12 +517,13 @@
   final RegionInfo regionInfo = regionReplicaInfo.getRegionInfo();
   final ServerName serverName = regionReplicaInfo.getServerName();
   final RegionState.State regionState = regionReplicaInfo.getRegionState();
-  final int rsPort = master.getRegionServerInfoPort(serverName);
 
   final long seqNum = regionReplicaInfo.getSeqNum();
 
   final String regionSpanFormat = "%s";
-  final String targetServerName = 
regionReplicaInfo.getTargetServerName().toString();
+  final String targetServerName = regionReplicaInfo.getTargetServerName() 
!= null
+? regionReplicaInfo.getTargetServerName().toString()
+: "";
   final Map mergeRegions = 
regionReplicaInfo.getMergeRegionInfo();
   final String mergeRegionNames = (mergeRegions == null) ? "" :
 mergeRegions.entrySet().stream()
@@ -540,7 +541,7 @@
   <%= endKeyDisplay %>
   <%= replicaIdDisplay %>
   <%= regionStateDisplay %>
-  "><%= 
buildRegionServerLink(serverName, rsPort, regionInfo, regionState) %>
+  "><%= 
serverName != null ? buildRegionServerLink(serverName, 
master.getRegionServerInfoPort(serverName), regionInfo, regionState) : "" 
%>
   <%= seqNum %>
   <%= targetServerName %>
   <%= mergeRegionNames %>

(hbase) branch branch-2.4 updated: HBASE-28191 Meta browser can happen NPE when server or targetServer of region is null (#5511)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.4 by this push:
 new 62f2383c700 HBASE-28191 Meta browser can happen NPE when server or 
targetServer of region is null (#5511)
62f2383c700 is described below

commit 62f2383c7000b4df171a908e346e62d01ce594d1
Author: guluo 
AuthorDate: Fri Nov 10 22:26:03 2023 +0800

HBASE-28191 Meta browser can happen NPE when server or targetServer of 
region is null (#5511)

Signed-off-by: Duo Zhang 
---
 hbase-server/src/main/resources/hbase-webapps/master/table.jsp | 7 ---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp 
b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
index 2b4b73fbdef..2f730ab76cc 100644
--- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
+++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
@@ -516,12 +516,13 @@
   final RegionInfo regionInfo = regionReplicaInfo.getRegionInfo();
   final ServerName serverName = regionReplicaInfo.getServerName();
   final RegionState.State regionState = regionReplicaInfo.getRegionState();
-  final int rsPort = master.getRegionServerInfoPort(serverName);
 
   final long seqNum = regionReplicaInfo.getSeqNum();
 
   final String regionSpanFormat = "%s";
-  final String targetServerName = 
regionReplicaInfo.getTargetServerName().toString();
+  final String targetServerName = regionReplicaInfo.getTargetServerName() 
!= null
+? regionReplicaInfo.getTargetServerName().toString()
+: "";
   final Map mergeRegions = 
regionReplicaInfo.getMergeRegionInfo();
   final String mergeRegionNames = (mergeRegions == null) ? "" :
 mergeRegions.entrySet().stream()
@@ -539,7 +540,7 @@
   <%= endKeyDisplay %>
   <%= replicaIdDisplay %>
   <%= regionStateDisplay %>
-  "><%= 
buildRegionServerLink(serverName, rsPort, regionInfo, regionState) %>
+  "><%= 
serverName != null ? buildRegionServerLink(serverName, 
master.getRegionServerInfoPort(serverName), regionInfo, regionState) : "" 
%>
   <%= seqNum %>
   <%= targetServerName %>
   <%= mergeRegionNames %>

(hbase) branch branch-2.5 updated: HBASE-28191 Meta browser can happen NPE when server or targetServer of region is null (#5512)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.5 by this push:
 new 218cb081968 HBASE-28191 Meta browser can happen NPE when server or 
targetServer of region is null (#5512)
218cb081968 is described below

commit 218cb08196827dd3ec914d251957af5118b424aa
Author: guluo 
AuthorDate: Fri Nov 10 22:25:48 2023 +0800

HBASE-28191 Meta browser can happen NPE when server or targetServer of 
region is null (#5512)

Signed-off-by: Duo Zhang 
---
 hbase-server/src/main/resources/hbase-webapps/master/table.jsp | 7 ---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp 
b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
index 84718edbc44..8b88f20301d 100644
--- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
+++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
@@ -504,12 +504,13 @@
   final RegionInfo regionInfo = regionReplicaInfo.getRegionInfo();
   final ServerName serverName = regionReplicaInfo.getServerName();
   final RegionState.State regionState = regionReplicaInfo.getRegionState();
-  final int rsPort = master.getRegionServerInfoPort(serverName);
 
   final long seqNum = regionReplicaInfo.getSeqNum();
 
   final String regionSpanFormat = "%s";
-  final String targetServerName = 
regionReplicaInfo.getTargetServerName().toString();
+  final String targetServerName = regionReplicaInfo.getTargetServerName() 
!= null
+? regionReplicaInfo.getTargetServerName().toString()
+: "";
   final Map mergeRegions = 
regionReplicaInfo.getMergeRegionInfo();
   final String mergeRegionNames = (mergeRegions == null) ? "" :
 mergeRegions.entrySet().stream()
@@ -527,7 +528,7 @@
   <%= endKeyDisplay %>
   <%= replicaIdDisplay %>
   <%= regionStateDisplay %>
-  "><%= 
buildRegionServerLink(serverName, rsPort, regionInfo, regionState) %>
+  "><%= 
serverName != null ? buildRegionServerLink(serverName, 
master.getRegionServerInfoPort(serverName), regionInfo, regionState) : "" 
%>
   <%= seqNum %>
   <%= targetServerName %>
   <%= mergeRegionNames %>

(hbase) branch branch-2 updated (2492da2abc7 -> 603b9263df7)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a change to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git


from 2492da2abc7 HBASE-27276 Reduce reflection overhead in Filter 
deserialization (#5488)
 add 603b9263df7 HBASE-28193 Update plugin for SBOM generation to 2.7.10 
(#5485)

No new revisions were added by this update.

Summary of changes:
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(hbase) branch branch-3 updated: HBASE-28193 Update plugin for SBOM generation to 2.7.10 (#5485)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-3 by this push:
 new 26934b785f7 HBASE-28193 Update plugin for SBOM generation to 2.7.10 
(#5485)
26934b785f7 is described below

commit 26934b785f7a01f09775d0b962d29157849943ff
Author: Vinod Anandan 
AuthorDate: Fri Nov 10 16:08:53 2023 +0200

HBASE-28193 Update plugin for SBOM generation to 2.7.10 (#5485)

Signed-off-by: Duo Zhang 
(cherry picked from commit e806350bd0956e5d981e49afba57f6591093ec6c)
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 6dbbd34b825..d2e1b59866b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -3492,7 +3492,7 @@
   
 org.cyclonedx
 cyclonedx-maven-plugin
-2.7.6
+2.7.10
 
   
 

(hbase) branch branch-2.5 updated: HBASE-28193 Update plugin for SBOM generation to 2.7.10 (#5485)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.5 by this push:
 new b43328234ec HBASE-28193 Update plugin for SBOM generation to 2.7.10 
(#5485)
b43328234ec is described below

commit b43328234ec794afbe789e59f10b7344b720ea3b
Author: Vinod Anandan 
AuthorDate: Fri Nov 10 16:08:53 2023 +0200

HBASE-28193 Update plugin for SBOM generation to 2.7.10 (#5485)

Signed-off-by: Duo Zhang 
(cherry picked from commit e806350bd0956e5d981e49afba57f6591093ec6c)
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index d8d48024a02..b54431668e3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -3172,7 +3172,7 @@
   
 org.cyclonedx
 cyclonedx-maven-plugin
-2.7.6
+2.7.10
 
   
 

(hbase) branch branch-2.4 updated: HBASE-28193 Update plugin for SBOM generation to 2.7.10 (#5485)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.4 by this push:
 new df4c47597fc HBASE-28193 Update plugin for SBOM generation to 2.7.10 
(#5485)
df4c47597fc is described below

commit df4c47597fccd63f3847a72b26bcc0ececc253e1
Author: Vinod Anandan 
AuthorDate: Fri Nov 10 16:08:53 2023 +0200

HBASE-28193 Update plugin for SBOM generation to 2.7.10 (#5485)

Signed-off-by: Duo Zhang 
(cherry picked from commit e806350bd0956e5d981e49afba57f6591093ec6c)
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 577bc97b738..80f166ce436 100755
--- a/pom.xml
+++ b/pom.xml
@@ -2983,7 +2983,7 @@
   
 org.cyclonedx
 cyclonedx-maven-plugin
-2.7.6
+2.7.10
 
   
 

(hbase) branch master updated (7151581f51f -> e806350bd09)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


from 7151581f51f HBASE-27276 Reduce reflection overhead in Filter 
deserialization (#5488)
 add e806350bd09 HBASE-28193 Update plugin for SBOM generation to 2.7.10 
(#5485)

No new revisions were added by this update.

Summary of changes:
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(hbase) branch branch-3 updated: HBASE-28122: Support TLSv1.3 cipher suites (#5444)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-3 by this push:
 new 34119d625fe HBASE-28122: Support TLSv1.3 cipher suites (#5444)
34119d625fe is described below

commit 34119d625fef133a0dff1b8aa57200477fe259f2
Author: Charles Connell 
AuthorDate: Wed Nov 8 20:03:01 2023 -0500

HBASE-28122: Support TLSv1.3 cipher suites (#5444)

Co-authored-by: Charles Connell 
Signed-off-by: Duo Zhang 
(cherry picked from commit d8b5198cfb50823577afd6a66c7fc5d401c825d9)
---
 .../hadoop/hbase/io/crypto/tls/X509Util.java   | 39 +-
 .../hadoop/hbase/io/crypto/tls/TestX509Util.java   |  8 ++---
 2 files changed, 28 insertions(+), 19 deletions(-)

diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/X509Util.java
 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/X509Util.java
index 7d16a82b1f3..41acfbbf48f 100644
--- 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/X509Util.java
+++ 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/X509Util.java
@@ -115,6 +115,10 @@ public final class X509Util {
 "hbase.client.netty.tls.handshaketimeout";
   public static final int DEFAULT_HANDSHAKE_DETECTION_TIMEOUT_MILLIS = 5000;
 
+  private static String[] getTls13Ciphers() {
+return new String[] { "TLS_AES_128_GCM_SHA256", "TLS_AES_256_GCM_SHA384" };
+  }
+
   private static String[] getGCMCiphers() {
 return new String[] { "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
   "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", 
"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
@@ -136,14 +140,17 @@ public final class X509Util {
   // Note that this performance assumption might not hold true for 
architectures other than x86_64.
   private static final String[] DEFAULT_CIPHERS_JAVA9 =
 ObjectArrays.concat(getGCMCiphers(), getCBCCiphers(), String.class);
+  private static final String[] DEFAULT_CIPHERS_JAVA11 =
+ObjectArrays.concat(ObjectArrays.concat(getTls13Ciphers(), 
getGCMCiphers(), String.class),
+  getCBCCiphers(), String.class);
 
   private static final String[] DEFAULT_CIPHERS_OPENSSL = 
getOpenSslFilteredDefaultCiphers();
 
   /**
* Not all of our default ciphers are available in OpenSSL. Takes our 
default cipher lists and
-   * filters them to only those available in OpenSsl. Does GCM first, then CBC 
because GCM tends to
-   * be better and faster, and we don't need to worry about the java8 vs 9 
performance issue if
-   * OpenSSL is handling it.
+   * filters them to only those available in OpenSsl. Prefers TLS 1.3, then 
GCM, then CBC because
+   * GCM tends to be better and faster, and we don't need to worry about the 
java8 vs 9 performance
+   * issue if OpenSSL is handling it.
*/
   private static String[] getOpenSslFilteredDefaultCiphers() {
 if (!OpenSsl.isAvailable()) {
@@ -152,16 +159,9 @@ public final class X509Util {
 
 Set openSslSuites = OpenSsl.availableJavaCipherSuites();
 List defaultSuites = new ArrayList<>();
-for (String cipher : getGCMCiphers()) {
-  if (openSslSuites.contains(cipher)) {
-defaultSuites.add(cipher);
-  }
-}
-for (String cipher : getCBCCiphers()) {
-  if (openSslSuites.contains(cipher)) {
-defaultSuites.add(cipher);
-  }
-}
+
Arrays.stream(getTls13Ciphers()).filter(openSslSuites::contains).forEach(defaultSuites::add);
+
Arrays.stream(getGCMCiphers()).filter(openSslSuites::contains).forEach(defaultSuites::add);
+
Arrays.stream(getCBCCiphers()).filter(openSslSuites::contains).forEach(defaultSuites::add);
 return defaultSuites.toArray(new String[0]);
   }
 
@@ -219,10 +219,19 @@ public final class X509Util {
 
   static String[] getDefaultCipherSuitesForJavaVersion(String javaVersion) {
 Objects.requireNonNull(javaVersion);
+
 if (javaVersion.matches("\\d+")) {
   // Must be Java 9 or later
-  LOG.debug("Using Java9+ optimized cipher suites for Java version {}", 
javaVersion);
-  return DEFAULT_CIPHERS_JAVA9;
+  int javaVersionInt = Integer.parseInt(javaVersion);
+  if (javaVersionInt >= 11) {
+LOG.debug(
+  "Using Java11+ optimized cipher suites for Java version {}, 
including TLSv1.3 support",
+  javaVersion);
+return DEFAULT_CIPHERS_JAVA11;
+  } else {
+LOG.debug("Using Java9+ optimized cipher suites for Java version {}", 
javaVersion);
+return DEFAULT_CIPHERS_JAVA9;
+  }
 } else if (javaVersion.startsWith("1.")) {
   // Must be Java 1.8 or earlier
   LOG.debug("Using Java8 optimized cipher suites for Java version {}", 
javaVersion);
diff --git 
a/hbase-common/src/test/

(hbase) branch branch-2 updated: HBASE-28122: Support TLSv1.3 cipher suites (#5444)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2 by this push:
 new 154b766a43c HBASE-28122: Support TLSv1.3 cipher suites (#5444)
154b766a43c is described below

commit 154b766a43c11a76686e6a2842ae382be92e6825
Author: Charles Connell 
AuthorDate: Wed Nov 8 20:03:01 2023 -0500

HBASE-28122: Support TLSv1.3 cipher suites (#5444)

Co-authored-by: Charles Connell 
Signed-off-by: Duo Zhang 
(cherry picked from commit d8b5198cfb50823577afd6a66c7fc5d401c825d9)
---
 .../hadoop/hbase/io/crypto/tls/X509Util.java   | 39 +-
 .../hadoop/hbase/io/crypto/tls/TestX509Util.java   |  8 ++---
 2 files changed, 28 insertions(+), 19 deletions(-)

diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/X509Util.java
 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/X509Util.java
index 8aae974d1ab..bfe22bd9378 100644
--- 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/X509Util.java
+++ 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/X509Util.java
@@ -115,6 +115,10 @@ public final class X509Util {
 "hbase.client.netty.tls.handshaketimeout";
   public static final int DEFAULT_HANDSHAKE_DETECTION_TIMEOUT_MILLIS = 5000;
 
+  private static String[] getTls13Ciphers() {
+return new String[] { "TLS_AES_128_GCM_SHA256", "TLS_AES_256_GCM_SHA384" };
+  }
+
   private static String[] getGCMCiphers() {
 return new String[] { "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
   "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", 
"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
@@ -136,14 +140,17 @@ public final class X509Util {
   // Note that this performance assumption might not hold true for 
architectures other than x86_64.
   private static final String[] DEFAULT_CIPHERS_JAVA9 =
 ObjectArrays.concat(getGCMCiphers(), getCBCCiphers(), String.class);
+  private static final String[] DEFAULT_CIPHERS_JAVA11 =
+ObjectArrays.concat(ObjectArrays.concat(getTls13Ciphers(), 
getGCMCiphers(), String.class),
+  getCBCCiphers(), String.class);
 
   private static final String[] DEFAULT_CIPHERS_OPENSSL = 
getOpenSslFilteredDefaultCiphers();
 
   /**
* Not all of our default ciphers are available in OpenSSL. Takes our 
default cipher lists and
-   * filters them to only those available in OpenSsl. Does GCM first, then CBC 
because GCM tends to
-   * be better and faster, and we don't need to worry about the java8 vs 9 
performance issue if
-   * OpenSSL is handling it.
+   * filters them to only those available in OpenSsl. Prefers TLS 1.3, then 
GCM, then CBC because
+   * GCM tends to be better and faster, and we don't need to worry about the 
java8 vs 9 performance
+   * issue if OpenSSL is handling it.
*/
   private static String[] getOpenSslFilteredDefaultCiphers() {
 if (!OpenSsl.isAvailable()) {
@@ -152,16 +159,9 @@ public final class X509Util {
 
 Set openSslSuites = OpenSsl.availableJavaCipherSuites();
 List defaultSuites = new ArrayList<>();
-for (String cipher : getGCMCiphers()) {
-  if (openSslSuites.contains(cipher)) {
-defaultSuites.add(cipher);
-  }
-}
-for (String cipher : getCBCCiphers()) {
-  if (openSslSuites.contains(cipher)) {
-defaultSuites.add(cipher);
-  }
-}
+
Arrays.stream(getTls13Ciphers()).filter(openSslSuites::contains).forEach(defaultSuites::add);
+
Arrays.stream(getGCMCiphers()).filter(openSslSuites::contains).forEach(defaultSuites::add);
+
Arrays.stream(getCBCCiphers()).filter(openSslSuites::contains).forEach(defaultSuites::add);
 return defaultSuites.toArray(new String[0]);
   }
 
@@ -219,10 +219,19 @@ public final class X509Util {
 
   static String[] getDefaultCipherSuitesForJavaVersion(String javaVersion) {
 Objects.requireNonNull(javaVersion);
+
 if (javaVersion.matches("\\d+")) {
   // Must be Java 9 or later
-  LOG.debug("Using Java9+ optimized cipher suites for Java version {}", 
javaVersion);
-  return DEFAULT_CIPHERS_JAVA9;
+  int javaVersionInt = Integer.parseInt(javaVersion);
+  if (javaVersionInt >= 11) {
+LOG.debug(
+  "Using Java11+ optimized cipher suites for Java version {}, 
including TLSv1.3 support",
+  javaVersion);
+return DEFAULT_CIPHERS_JAVA11;
+  } else {
+LOG.debug("Using Java9+ optimized cipher suites for Java version {}", 
javaVersion);
+return DEFAULT_CIPHERS_JAVA9;
+  }
 } else if (javaVersion.startsWith("1.")) {
   // Must be Java 1.8 or earlier
   LOG.debug("Using Java8 optimized cipher suites for Java version {}", 
javaVersion);
diff --git 
a/hbase-common/src/test/

(hbase) branch master updated: HBASE-28122: Support TLSv1.3 cipher suites (#5444)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
 new d8b5198cfb5 HBASE-28122: Support TLSv1.3 cipher suites (#5444)
d8b5198cfb5 is described below

commit d8b5198cfb50823577afd6a66c7fc5d401c825d9
Author: Charles Connell 
AuthorDate: Wed Nov 8 20:03:01 2023 -0500

HBASE-28122: Support TLSv1.3 cipher suites (#5444)

Co-authored-by: Charles Connell 
Signed-off-by: Duo Zhang 
---
 .../hadoop/hbase/io/crypto/tls/X509Util.java   | 39 +-
 .../hadoop/hbase/io/crypto/tls/TestX509Util.java   |  8 ++---
 2 files changed, 28 insertions(+), 19 deletions(-)

diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/X509Util.java
 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/X509Util.java
index 7d16a82b1f3..41acfbbf48f 100644
--- 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/X509Util.java
+++ 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/X509Util.java
@@ -115,6 +115,10 @@ public final class X509Util {
 "hbase.client.netty.tls.handshaketimeout";
   public static final int DEFAULT_HANDSHAKE_DETECTION_TIMEOUT_MILLIS = 5000;
 
+  private static String[] getTls13Ciphers() {
+return new String[] { "TLS_AES_128_GCM_SHA256", "TLS_AES_256_GCM_SHA384" };
+  }
+
   private static String[] getGCMCiphers() {
 return new String[] { "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
   "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", 
"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
@@ -136,14 +140,17 @@ public final class X509Util {
   // Note that this performance assumption might not hold true for 
architectures other than x86_64.
   private static final String[] DEFAULT_CIPHERS_JAVA9 =
 ObjectArrays.concat(getGCMCiphers(), getCBCCiphers(), String.class);
+  private static final String[] DEFAULT_CIPHERS_JAVA11 =
+ObjectArrays.concat(ObjectArrays.concat(getTls13Ciphers(), 
getGCMCiphers(), String.class),
+  getCBCCiphers(), String.class);
 
   private static final String[] DEFAULT_CIPHERS_OPENSSL = 
getOpenSslFilteredDefaultCiphers();
 
   /**
* Not all of our default ciphers are available in OpenSSL. Takes our 
default cipher lists and
-   * filters them to only those available in OpenSsl. Does GCM first, then CBC 
because GCM tends to
-   * be better and faster, and we don't need to worry about the java8 vs 9 
performance issue if
-   * OpenSSL is handling it.
+   * filters them to only those available in OpenSsl. Prefers TLS 1.3, then 
GCM, then CBC because
+   * GCM tends to be better and faster, and we don't need to worry about the 
java8 vs 9 performance
+   * issue if OpenSSL is handling it.
*/
   private static String[] getOpenSslFilteredDefaultCiphers() {
 if (!OpenSsl.isAvailable()) {
@@ -152,16 +159,9 @@ public final class X509Util {
 
 Set openSslSuites = OpenSsl.availableJavaCipherSuites();
 List defaultSuites = new ArrayList<>();
-for (String cipher : getGCMCiphers()) {
-  if (openSslSuites.contains(cipher)) {
-defaultSuites.add(cipher);
-  }
-}
-for (String cipher : getCBCCiphers()) {
-  if (openSslSuites.contains(cipher)) {
-defaultSuites.add(cipher);
-  }
-}
+
Arrays.stream(getTls13Ciphers()).filter(openSslSuites::contains).forEach(defaultSuites::add);
+
Arrays.stream(getGCMCiphers()).filter(openSslSuites::contains).forEach(defaultSuites::add);
+
Arrays.stream(getCBCCiphers()).filter(openSslSuites::contains).forEach(defaultSuites::add);
 return defaultSuites.toArray(new String[0]);
   }
 
@@ -219,10 +219,19 @@ public final class X509Util {
 
   static String[] getDefaultCipherSuitesForJavaVersion(String javaVersion) {
 Objects.requireNonNull(javaVersion);
+
 if (javaVersion.matches("\\d+")) {
   // Must be Java 9 or later
-  LOG.debug("Using Java9+ optimized cipher suites for Java version {}", 
javaVersion);
-  return DEFAULT_CIPHERS_JAVA9;
+  int javaVersionInt = Integer.parseInt(javaVersion);
+  if (javaVersionInt >= 11) {
+LOG.debug(
+  "Using Java11+ optimized cipher suites for Java version {}, 
including TLSv1.3 support",
+  javaVersion);
+return DEFAULT_CIPHERS_JAVA11;
+  } else {
+LOG.debug("Using Java9+ optimized cipher suites for Java version {}", 
javaVersion);
+return DEFAULT_CIPHERS_JAVA9;
+  }
 } else if (javaVersion.startsWith("1.")) {
   // Must be Java 1.8 or earlier
   LOG.debug("Using Java8 optimized cipher suites for Java version {}", 
javaVersion);
diff --git 
a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/tls/TestX509Util.java
 
b/hbase-common/sr

(hbase) branch branch-3 updated: HBASE-28191 Meta browser can happen NPE when server or targetServer of region is null (#5508)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-3 by this push:
 new 0c40defe349 HBASE-28191 Meta browser can happen NPE when server or 
targetServer of region is null (#5508)
0c40defe349 is described below

commit 0c40defe3490a4133b3eee37659b1e061cfb408d
Author: guluo 
AuthorDate: Wed Nov 8 22:02:09 2023 +0800

HBASE-28191 Meta browser can happen NPE when server or targetServer of 
region is null (#5508)

Signed-off-by: Wellington Chevreuil 
Signed-off-by: Duo Zhang 
(cherry picked from commit 954a1f8fc392d19056cf11b896a188cda462c43b)
---
 hbase-server/src/main/resources/hbase-webapps/master/table.jsp | 7 ---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp 
b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
index 296e0e77f73..1d48a7561e1 100644
--- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
+++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
@@ -562,12 +562,13 @@
   final RegionInfo regionInfo = regionReplicaInfo.getRegionInfo();
   final ServerName serverName = regionReplicaInfo.getServerName();
   final RegionState.State regionState = regionReplicaInfo.getRegionState();
-  final int rsPort = master.getRegionServerInfoPort(serverName);
 
   final long seqNum = regionReplicaInfo.getSeqNum();
 
   final String regionSpanFormat = "%s";
-  final String targetServerName = 
regionReplicaInfo.getTargetServerName().toString();
+  final String targetServerName = regionReplicaInfo.getTargetServerName() 
!= null
+? regionReplicaInfo.getTargetServerName().toString()
+: "";
   final Map mergeRegions = 
regionReplicaInfo.getMergeRegionInfo();
   final String mergeRegionNames = (mergeRegions == null) ? "" :
 mergeRegions.entrySet().stream()
@@ -585,7 +586,7 @@
   <%= endKeyDisplay %>
   <%= replicaIdDisplay %>
   <%= regionStateDisplay %>
-  "><%= 
buildRegionLink(serverName, rsPort, regionInfo, regionState) %>
+  "><%= 
serverName != null ? buildRegionLink(serverName, 
master.getRegionServerInfoPort(serverName), regionInfo, regionState) : "" 
%>
   <%= seqNum %>
   <%= targetServerName %>
   <%= mergeRegionNames %>

(hbase) branch master updated: HBASE-28191 Meta browser can happen NPE when server or targetServer of region is null (#5508)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
 new 954a1f8fc39 HBASE-28191 Meta browser can happen NPE when server or 
targetServer of region is null (#5508)
954a1f8fc39 is described below

commit 954a1f8fc392d19056cf11b896a188cda462c43b
Author: guluo 
AuthorDate: Wed Nov 8 22:02:09 2023 +0800

HBASE-28191 Meta browser can happen NPE when server or targetServer of 
region is null (#5508)

Signed-off-by: Wellington Chevreuil 
Signed-off-by: Duo Zhang 
---
 hbase-server/src/main/resources/hbase-webapps/master/table.jsp | 7 ---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp 
b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
index 296e0e77f73..1d48a7561e1 100644
--- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
+++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
@@ -562,12 +562,13 @@
   final RegionInfo regionInfo = regionReplicaInfo.getRegionInfo();
   final ServerName serverName = regionReplicaInfo.getServerName();
   final RegionState.State regionState = regionReplicaInfo.getRegionState();
-  final int rsPort = master.getRegionServerInfoPort(serverName);
 
   final long seqNum = regionReplicaInfo.getSeqNum();
 
   final String regionSpanFormat = "%s";
-  final String targetServerName = 
regionReplicaInfo.getTargetServerName().toString();
+  final String targetServerName = regionReplicaInfo.getTargetServerName() 
!= null
+? regionReplicaInfo.getTargetServerName().toString()
+: "";
   final Map mergeRegions = 
regionReplicaInfo.getMergeRegionInfo();
   final String mergeRegionNames = (mergeRegions == null) ? "" :
 mergeRegions.entrySet().stream()
@@ -585,7 +586,7 @@
   <%= endKeyDisplay %>
   <%= replicaIdDisplay %>
   <%= regionStateDisplay %>
-  "><%= 
buildRegionLink(serverName, rsPort, regionInfo, regionState) %>
+  "><%= 
serverName != null ? buildRegionLink(serverName, 
master.getRegionServerInfoPort(serverName), regionInfo, regionState) : "" 
%>
   <%= seqNum %>
   <%= targetServerName %>
   <%= mergeRegionNames %>

(hbase) branch branch-3 updated: HBASE-28185 Alter table to set TTL using hbase shell failed when ttl string is not match format (#5494)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-3 by this push:
 new e19fb8f740d HBASE-28185 Alter table to set TTL using hbase shell 
failed when ttl string is not match format (#5494)
e19fb8f740d is described below

commit e19fb8f740dc81921195a765b59c14a080b6e112
Author: chaijunjie0101 <64140218+chaijunjie0...@users.noreply.github.com>
AuthorDate: Mon Nov 6 10:32:19 2023 +0800

HBASE-28185 Alter table to set TTL using hbase shell failed when ttl string 
is not match format (#5494)

Signed-off-by: Duo Zhang 
(cherry picked from commit 027a119bcf495e18b3e0fa4984f8982b9ed1390d)
---
 .../src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java | 4 
 1 file changed, 4 insertions(+)

diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java
index f73064f70a8..1b19bd25287 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java
@@ -184,7 +184,11 @@ public final class PrettyPrinter {
   hours = matcher.group(6);
   minutes = matcher.group(8);
   seconds = matcher.group(10);
+} else {
+  LOG.warn("Given interval value '{}' is not a number and does not match 
human readable format,"
++ " value will be set to 0.", humanReadableInterval);
 }
+
 ttl = 0;
 ttl += days != null ? Long.parseLong(days) * HConstants.DAY_IN_SECONDS : 0;
 ttl += hours != null ? Long.parseLong(hours) * HConstants.HOUR_IN_SECONDS 
: 0;

(hbase) branch branch-2 updated: HBASE-28185 Alter table to set TTL using hbase shell failed when ttl string is not match format (#5494)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2 by this push:
 new c0b5e962c7a HBASE-28185 Alter table to set TTL using hbase shell 
failed when ttl string is not match format (#5494)
c0b5e962c7a is described below

commit c0b5e962c7afe70590d0702ab7d8f3edb0128ca2
Author: chaijunjie0101 <64140218+chaijunjie0...@users.noreply.github.com>
AuthorDate: Mon Nov 6 10:32:19 2023 +0800

HBASE-28185 Alter table to set TTL using hbase shell failed when ttl string 
is not match format (#5494)

Signed-off-by: Duo Zhang 
(cherry picked from commit 027a119bcf495e18b3e0fa4984f8982b9ed1390d)
---
 .../src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java | 4 
 1 file changed, 4 insertions(+)

diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java
index f73064f70a8..1b19bd25287 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java
@@ -184,7 +184,11 @@ public final class PrettyPrinter {
   hours = matcher.group(6);
   minutes = matcher.group(8);
   seconds = matcher.group(10);
+} else {
+  LOG.warn("Given interval value '{}' is not a number and does not match 
human readable format,"
++ " value will be set to 0.", humanReadableInterval);
 }
+
 ttl = 0;
 ttl += days != null ? Long.parseLong(days) * HConstants.DAY_IN_SECONDS : 0;
 ttl += hours != null ? Long.parseLong(hours) * HConstants.HOUR_IN_SECONDS 
: 0;

(hbase) branch branch-2.5 updated: HBASE-28185 Alter table to set TTL using hbase shell failed when ttl string is not match format (#5494)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.5 by this push:
 new 4236b70070c HBASE-28185 Alter table to set TTL using hbase shell 
failed when ttl string is not match format (#5494)
4236b70070c is described below

commit 4236b70070ce2ba8edf4358c4738d6c8cdb64c2b
Author: chaijunjie0101 <64140218+chaijunjie0...@users.noreply.github.com>
AuthorDate: Mon Nov 6 10:32:19 2023 +0800

HBASE-28185 Alter table to set TTL using hbase shell failed when ttl string 
is not match format (#5494)

Signed-off-by: Duo Zhang 
(cherry picked from commit 027a119bcf495e18b3e0fa4984f8982b9ed1390d)
---
 .../src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java | 4 
 1 file changed, 4 insertions(+)

diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java
index f73064f70a8..1b19bd25287 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java
@@ -184,7 +184,11 @@ public final class PrettyPrinter {
   hours = matcher.group(6);
   minutes = matcher.group(8);
   seconds = matcher.group(10);
+} else {
+  LOG.warn("Given interval value '{}' is not a number and does not match 
human readable format,"
++ " value will be set to 0.", humanReadableInterval);
 }
+
 ttl = 0;
 ttl += days != null ? Long.parseLong(days) * HConstants.DAY_IN_SECONDS : 0;
 ttl += hours != null ? Long.parseLong(hours) * HConstants.HOUR_IN_SECONDS 
: 0;

(hbase) branch branch-2.4 updated: HBASE-28185 Alter table to set TTL using hbase shell failed when ttl string is not match format (#5494)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.4 by this push:
 new e6f269b9b3e HBASE-28185 Alter table to set TTL using hbase shell 
failed when ttl string is not match format (#5494)
e6f269b9b3e is described below

commit e6f269b9b3e7f341ec40e4196cb334eb577d2c8a
Author: chaijunjie0101 <64140218+chaijunjie0...@users.noreply.github.com>
AuthorDate: Mon Nov 6 10:32:19 2023 +0800

HBASE-28185 Alter table to set TTL using hbase shell failed when ttl string 
is not match format (#5494)

Signed-off-by: Duo Zhang 
(cherry picked from commit 027a119bcf495e18b3e0fa4984f8982b9ed1390d)
---
 .../src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java | 4 
 1 file changed, 4 insertions(+)

diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java
index d5b3f894a7f..516f788f0b8 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java
@@ -173,7 +173,11 @@ public final class PrettyPrinter {
   hours = matcher.group(6);
   minutes = matcher.group(8);
   seconds = matcher.group(10);
+} else {
+  LOG.warn("Given interval value '{}' is not a number and does not match 
human readable format,"
++ " value will be set to 0.", humanReadableInterval);
 }
+
 ttl = 0;
 ttl += days != null ? Long.parseLong(days) * HConstants.DAY_IN_SECONDS : 0;
 ttl += hours != null ? Long.parseLong(hours) * HConstants.HOUR_IN_SECONDS 
: 0;

(hbase) branch master updated: HBASE-28185 Alter table to set TTL using hbase shell failed when ttl string is not match format (#5494)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
 new 027a119bcf4 HBASE-28185 Alter table to set TTL using hbase shell 
failed when ttl string is not match format (#5494)
027a119bcf4 is described below

commit 027a119bcf495e18b3e0fa4984f8982b9ed1390d
Author: chaijunjie0101 <64140218+chaijunjie0...@users.noreply.github.com>
AuthorDate: Mon Nov 6 10:32:19 2023 +0800

HBASE-28185 Alter table to set TTL using hbase shell failed when ttl string 
is not match format (#5494)

Signed-off-by: Duo Zhang 
---
 .../src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java | 4 
 1 file changed, 4 insertions(+)

diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java
index f73064f70a8..1b19bd25287 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java
@@ -184,7 +184,11 @@ public final class PrettyPrinter {
   hours = matcher.group(6);
   minutes = matcher.group(8);
   seconds = matcher.group(10);
+} else {
+  LOG.warn("Given interval value '{}' is not a number and does not match 
human readable format,"
++ " value will be set to 0.", humanReadableInterval);
 }
+
 ttl = 0;
 ttl += days != null ? Long.parseLong(days) * HConstants.DAY_IN_SECONDS : 0;
 ttl += hours != null ? Long.parseLong(hours) * HConstants.HOUR_IN_SECONDS 
: 0;

(hbase) branch branch-2.4 updated: HBASE-28153 Upgrade zookeeper to a newer version (#5484) (#5490)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.4 by this push:
 new 373b8b88670 HBASE-28153 Upgrade zookeeper to a newer version (#5484) 
(#5490)
373b8b88670 is described below

commit 373b8b88670624c46cb1587acc95b309148f343d
Author: Duo Zhang 
AuthorDate: Thu Nov 2 09:40:16 2023 +0800

HBASE-28153 Upgrade zookeeper to a newer version (#5484) (#5490)

Backport of #5475

(cherry picked from commit 0d04a6053a05525e0ba8fd6525c3af7f3412c9de)

Signed-off-by: Andrew Purtell 
---
 hbase-it/pom.xml|   7 --
 hbase-server/pom.xml|   8 +-
 hbase-zookeeper/pom.xml |   9 ++
 pom.xml | 245 
 4 files changed, 94 insertions(+), 175 deletions(-)

diff --git a/hbase-it/pom.xml b/hbase-it/pom.xml
index 2eb21038a73..30e5fdbdc9a 100644
--- a/hbase-it/pom.xml
+++ b/hbase-it/pom.xml
@@ -151,13 +151,6 @@
   org.apache.htrace
   htrace-core4
 
-
-
-  io.netty
-  netty
-  ${netty.hadoop.version}
-  test
-
 
   org.slf4j
   jcl-over-slf4j
diff --git a/hbase-server/pom.xml b/hbase-server/pom.xml
index 3487b0b5d10..86d878ec2c4 100644
--- a/hbase-server/pom.xml
+++ b/hbase-server/pom.xml
@@ -803,13 +803,11 @@
 
   
 
-
+
   
   
 
diff --git a/hbase-zookeeper/pom.xml b/hbase-zookeeper/pom.xml
index 698610f3df1..8fa3318a0a4 100644
--- a/hbase-zookeeper/pom.xml
+++ b/hbase-zookeeper/pom.xml
@@ -134,6 +134,15 @@
   reload4j
   test
 
+
+
+  org.xerial.snappy
+  snappy-java
+
+
+  commons-cli
+  commons-cli
+
   
 
   
diff --git a/pom.xml b/pom.xml
index 5ad0c76e8aa..577bc97b738 100755
--- a/pom.xml
+++ b/pom.xml
@@ -549,9 +549,18 @@
 11.0.2
 hbase-hadoop2-compat
 src/main/assembly/hadoop-two-compat.xml
-
-3.6.2.Final
+
+3.10.6.Final
+4.1.100.Final
 
 0.13.0
 
 0.8.8
 
@@ -1105,6 +1116,11 @@
 commons-math3
 ${commons-math.version}
   
+  
+commons-cli
+commons-cli
+${commons-cli.version}
+  
   
 
-  
-io.netty
-netty-all
-  
   
 javax.servlet
 servlet-api
@@ -3695,14 +3623,6 @@
 com.sun.jersey
 jersey-core
   
-  
-io.netty
-netty
-  
-  
-io.netty
-netty-all
-  
   
 javax.servlet
 servlet-api
@@ -3811,12 +3731,20 @@
 log4j
   
   
-io.netty
-netty
+ch.qos.reload4j
+reload4j
   
   
-io.netty
-netty-all
+org.slf4j
+slf4j-reload4j
+  
+  
+org.fusesource.leveldbjni
+leveldbjni-all
+  
+  
+org.openlabtesting.leveldbjni
+leveldbjni-all
   
 
   
@@ -3868,22 +3796,12 @@
 log4j
   
   
-io.netty
-netty-all
+ch.qos.reload4j
+reload4j
   
-
-  
-  
-org.apache.hadoop
-hadoop-hdfs
-${hadoop-three.version}
-tests
-test-jar
-test
-
   
-com.sun.jersey
-jersey-core
+org.slf4j
+slf4j-reload4j
   
 
   
@@ -3943,14 +3861,6 @@
 stax
 stax-api
   
-  
-io.netty
-netty
-  
-  
-io.netty
-netty-all
-  
   
 com.google.code.findbugs
 jsr305
@@ -4033,10 +3943,6 @@
 com.google.code.findbugs
 jsr305
   
-  
-io.netty
-netty
-  
   
 org.slf4j
 slf4j-log4j12
@@ -4045,6 +3951,30 @@
 log4j
 log4j
   
+  
+ch.qos.reload4j
+reload4j
+  
+  
+org.slf4j
+slf4j-reload4j
+  
+  
+org.codehaus.jackson

(hbase) branch branch-2.5 updated: HBASE-28153 Upgrade zookeeper to a newer version (#5484) (#5489)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.5 by this push:
 new 7639345a970 HBASE-28153 Upgrade zookeeper to a newer version (#5484) 
(#5489)
7639345a970 is described below

commit 7639345a970636e7a9eb7adf6d84dadd6f3fccb9
Author: Duo Zhang 
AuthorDate: Thu Nov 2 09:31:38 2023 +0800

HBASE-28153 Upgrade zookeeper to a newer version (#5484) (#5489)

Backport of #5475

(cherry picked from commit 0d04a6053a05525e0ba8fd6525c3af7f3412c9de)

Signed-off-by: Andrew Purtell 
---
 hbase-it/pom.xml|   7 --
 hbase-server/pom.xml|   7 --
 hbase-zookeeper/pom.xml |   9 +++
 pom.xml | 204 +---
 4 files changed, 45 insertions(+), 182 deletions(-)

diff --git a/hbase-it/pom.xml b/hbase-it/pom.xml
index 8539093a79b..de0e0e1a291 100644
--- a/hbase-it/pom.xml
+++ b/hbase-it/pom.xml
@@ -151,13 +151,6 @@
   io.opentelemetry
   opentelemetry-api
 
-
-
-  io.netty
-  netty
-  ${netty.hadoop.version}
-  test
-
 
   org.slf4j
   jcl-over-slf4j
diff --git a/hbase-server/pom.xml b/hbase-server/pom.xml
index 0ad01b84cae..24bea0f07f5 100644
--- a/hbase-server/pom.xml
+++ b/hbase-server/pom.xml
@@ -830,13 +830,6 @@
 
   
 
-
 
   org.apache.hadoop
   hadoop-minikdc
diff --git a/hbase-zookeeper/pom.xml b/hbase-zookeeper/pom.xml
index 9fac0ea9feb..47ba0da77dc 100644
--- a/hbase-zookeeper/pom.xml
+++ b/hbase-zookeeper/pom.xml
@@ -143,6 +143,15 @@
   log4j-slf4j-impl
   test
 
+
+
+  org.xerial.snappy
+  snappy-java
+
+
+  commons-cli
+  commons-cli
+
   
 
   
diff --git a/pom.xml b/pom.xml
index d3215f82952..d8d48024a02 100644
--- a/pom.xml
+++ b/pom.xml
@@ -554,10 +554,18 @@
 11.0.2
 hbase-hadoop2-compat
 src/main/assembly/hadoop-two-compat.xml
-
-3.10.6.Final
+
+3.10.6.Final
+4.1.100.Final
 
 0.13.0
 
-  
-io.netty
-netty-all
-  
   
 javax.servlet
 servlet-api
@@ -4026,14 +3947,6 @@
 com.sun.jersey
 jersey-core
   
-  
-io.netty
-netty
-  
-  
-io.netty
-netty-all
-  
   
 javax.servlet
 servlet-api
@@ -4153,14 +4066,6 @@
 org.slf4j
 slf4j-reload4j
   
-  
-io.netty
-netty
-  
-  
-io.netty
-netty-all
-  
   
 org.fusesource.leveldbjni
 leveldbjni-all
@@ -4222,13 +4127,6 @@
 org.slf4j
 slf4j-reload4j
   
-  
 
   
   
@@ -4295,14 +4193,6 @@
 stax
 stax-api
   
-  
-io.netty
-netty
-  
-  
-io.netty
-netty-all
-  
   
 com.google.code.findbugs
 jsr305
@@ -4389,10 +4279,6 @@
 com.google.code.findbugs
 jsr305
   
-  
-io.netty
-netty
-  
   
 org.slf4j
 slf4j-log4j12
@@ -4409,13 +4295,6 @@
 org.slf4j
 slf4j-reload4j
   
-  
   
 org.codehaus.jackson
 *
@@ -4479,17 +4358,6 @@
 stax
 stax-api
   
-  
-io.netty
-netty
-  
-  
   
 com.google.code.findbugs
 jsr305

(hbase) branch branch-3 updated: HBASE-28153 Upgrade zookeeper to a newer version (#5475)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-3 by this push:
 new 6cd718a6d55 HBASE-28153 Upgrade zookeeper to a newer version (#5475)
6cd718a6d55 is described below

commit 6cd718a6d551361b67cfc1da0b154ac25f48b460
Author: Duo Zhang 
AuthorDate: Wed Nov 1 10:46:32 2023 +0800

HBASE-28153 Upgrade zookeeper to a newer version (#5475)

Signed-off-by: Nick Dimiduk 
Signed-off-by: Andrew Purtell 
Signed-off-by: Bryan Beaudreault 
(cherry picked from commit a97373965e3aca45e0a43d38d6de2827c8f4c4ae)
---
 hbase-it/pom.xml|   7 ---
 hbase-server/pom.xml|   7 ---
 hbase-zookeeper/pom.xml |   9 
 pom.xml | 133 +---
 4 files changed, 44 insertions(+), 112 deletions(-)

diff --git a/hbase-it/pom.xml b/hbase-it/pom.xml
index bc67f21c7dc..c3f884a16b0 100644
--- a/hbase-it/pom.xml
+++ b/hbase-it/pom.xml
@@ -132,13 +132,6 @@
   io.opentelemetry
   opentelemetry-api
 
-
-
-  io.netty
-  netty
-  ${netty.hadoop.version}
-  test
-
 
   org.slf4j
   jcl-over-slf4j
diff --git a/hbase-server/pom.xml b/hbase-server/pom.xml
index 2455c199cb2..c04947529bf 100644
--- a/hbase-server/pom.xml
+++ b/hbase-server/pom.xml
@@ -699,13 +699,6 @@
 
   
 
-
 
   org.apache.hadoop
   hadoop-minikdc
diff --git a/hbase-zookeeper/pom.xml b/hbase-zookeeper/pom.xml
index f1711fdf338..8fceb26d5b0 100644
--- a/hbase-zookeeper/pom.xml
+++ b/hbase-zookeeper/pom.xml
@@ -139,6 +139,15 @@
   log4j-slf4j-impl
   test
 
+
+
+  org.xerial.snappy
+  snappy-java
+
+
+  commons-cli
+  commons-cli
+
   
 
   
diff --git a/pom.xml b/pom.xml
index 277baa3cddb..6dbbd34b825 100644
--- a/pom.xml
+++ b/pom.xml
@@ -802,9 +802,18 @@
 -->
 ${hadoop-three.version}
 src/main/assembly/hadoop-three-compat.xml
-
-3.10.5.Final
+
+3.10.6.Final
+4.1.100.Final
 
 0.13.0
 
   
 com.google.code.findbugs
 jsr305
@@ -3935,14 +3895,6 @@
 org.slf4j
 slf4j-reload4j
   
-  
-io.netty
-netty
-  
-  
-io.netty
-netty-all
-  
 
   
   
@@ -3984,13 +3936,6 @@
 org.slf4j
 slf4j-reload4j
   
-  
   
 org.codehaus.jackson
 *
@@ -4056,14 +4001,6 @@
 stax
 stax-api
   
-  
-io.netty
-netty
-  
-  
-io.netty
-netty-all
-  
   
 com.google.code.findbugs
 jsr305

(hbase) branch branch-2 updated: HBASE-28153 Upgrade zookeeper to a newer version (#5484)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2 by this push:
 new 0d04a6053a0 HBASE-28153 Upgrade zookeeper to a newer version (#5484)
0d04a6053a0 is described below

commit 0d04a6053a05525e0ba8fd6525c3af7f3412c9de
Author: Duo Zhang 
AuthorDate: Wed Nov 1 10:48:01 2023 +0800

HBASE-28153 Upgrade zookeeper to a newer version (#5484)

Backport of #5475
---
 hbase-backup/pom.xml|   5 ++
 hbase-it/pom.xml|   7 --
 hbase-server/pom.xml|   7 --
 hbase-zookeeper/pom.xml |   9 +++
 pom.xml | 204 +---
 5 files changed, 50 insertions(+), 182 deletions(-)

diff --git a/hbase-backup/pom.xml b/hbase-backup/pom.xml
index 176746d2cda..be057a4c3c5 100644
--- a/hbase-backup/pom.xml
+++ b/hbase-backup/pom.xml
@@ -43,6 +43,11 @@
   test-jar
   test
 
+
+  javax.ws.rs
+  javax.ws.rs-api
+  test
+
 
   org.apache.hbase
   hbase-client
diff --git a/hbase-it/pom.xml b/hbase-it/pom.xml
index 96766fa5e56..180bb076668 100644
--- a/hbase-it/pom.xml
+++ b/hbase-it/pom.xml
@@ -155,13 +155,6 @@
   io.opentelemetry
   opentelemetry-api
 
-
-
-  io.netty
-  netty
-  ${netty.hadoop.version}
-  test
-
 
   org.slf4j
   jcl-over-slf4j
diff --git a/hbase-server/pom.xml b/hbase-server/pom.xml
index f27c4112536..132e1faa6ca 100644
--- a/hbase-server/pom.xml
+++ b/hbase-server/pom.xml
@@ -835,13 +835,6 @@
 
   
 
-
 
   org.apache.hadoop
   hadoop-minikdc
diff --git a/hbase-zookeeper/pom.xml b/hbase-zookeeper/pom.xml
index 19eb33dfb55..f2b997b09f1 100644
--- a/hbase-zookeeper/pom.xml
+++ b/hbase-zookeeper/pom.xml
@@ -143,6 +143,15 @@
   log4j-slf4j-impl
   test
 
+
+
+  org.xerial.snappy
+  snappy-java
+
+
+  commons-cli
+  commons-cli
+
   
 
   
diff --git a/pom.xml b/pom.xml
index bd50ebda011..414a155fa9b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -555,10 +555,18 @@
 11.0.2
 hbase-hadoop2-compat
 src/main/assembly/hadoop-two-compat.xml
-
-3.10.6.Final
+
+3.10.6.Final
+4.1.100.Final
 
 0.13.0
 
-  
-io.netty
-netty-all
-  
   
 javax.servlet
 servlet-api
@@ -4039,14 +3960,6 @@
 com.sun.jersey
 jersey-core
   
-  
-io.netty
-netty
-  
-  
-io.netty
-netty-all
-  
   
 javax.servlet
 servlet-api
@@ -4166,14 +4079,6 @@
 org.slf4j
 slf4j-reload4j
   
-  
-io.netty
-netty
-  
-  
-io.netty
-netty-all
-  
   
 org.fusesource.leveldbjni
 leveldbjni-all
@@ -4235,13 +4140,6 @@
 org.slf4j
 slf4j-reload4j
   
-  
 
   
   
@@ -4308,14 +4206,6 @@
 stax
 stax-api
   
-  
-io.netty
-netty
-  
-  
-io.netty
-netty-all
-  
   
 com.google.code.findbugs
 jsr305
@@ -4402,10 +4292,6 @@
 com.google.code.findbugs
 jsr305
   
-  
-io.netty
-netty
-  
   
 org.slf4j
 slf4j-log4j12
@@ -4422,13 +4308,6 @@
 org.slf4j
 slf4j-reload4j
   
-  
   
 org.codehaus.jackson
 *
@@ -4492,17 +4371,6 @@
 stax
 stax-api
   
-  
-io.netty
-netty
-  
-  
   
 com.google.code.findbugs
 jsr305

(hbase) branch master updated (208e9b1a828 -> a97373965e3)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


from 208e9b1a828 HBASE-28164 Add hbase-connectos 1.0.1 to download page 
(#5483)
 add a97373965e3 HBASE-28153 Upgrade zookeeper to a newer version (#5475)

No new revisions were added by this update.

Summary of changes:
 hbase-it/pom.xml|   7 ---
 hbase-server/pom.xml|   7 ---
 hbase-zookeeper/pom.xml |   9 
 pom.xml | 133 +---
 4 files changed, 44 insertions(+), 112 deletions(-)

[hbase] branch branch-3 updated: HBASE-28145 When specifying the wrong BoolFilter type while creating a table in HBase shell, the log prompt will report an error. (#5460)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-3 by this push:
 new 68ff65c5169 HBASE-28145 When specifying the wrong BoolFilter type 
while creating a table in HBase shell, the log prompt will report an error. 
(#5460)
68ff65c5169 is described below

commit 68ff65c51691bcc854d6cd3aec4258109abfb8c0
Author: ZhangIssac <58984599+zhangis...@users.noreply.github.com>
AuthorDate: Wed Oct 25 15:08:41 2023 +0800

HBASE-28145 When specifying the wrong BoolFilter type while creating a 
table in HBase shell, the log prompt will report an error. (#5460)

Co-authored-by: xiaozhang 
Signed-off-by: Duo Zhang 
(cherry picked from commit 13d46e7f8cd488d9d352baef4eacb0d8681d0295)
---
 hbase-shell/src/main/ruby/hbase/admin.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb 
b/hbase-shell/src/main/ruby/hbase/admin.rb
index 53a8137fc0c..453b7ae1af6 100644
--- a/hbase-shell/src/main/ruby/hbase/admin.rb
+++ b/hbase-shell/src/main/ruby/hbase/admin.rb
@@ -1182,7 +1182,7 @@ module Hbase
 if 
org.apache.hadoop.hbase.regionserver.BloomType.constants.include?(bloomtype)
   
cfdb.setBloomFilterType(org.apache.hadoop.hbase.regionserver.BloomType.valueOf(bloomtype))
 else
-  raise(ArgumentError, "BloomFilter type #{bloomtype} is not 
supported. Use one of " + 
org.apache.hadoop.hbase.regionserver.StoreFile::BloomType.constants.join(' '))
+  raise(ArgumentError, "BloomFilter type #{bloomtype} is not 
supported. Use one of " + 
org.apache.hadoop.hbase.regionserver.BloomType.constants.join(' '))
 end
   end
   if arg.include?(ColumnFamilyDescriptorBuilder::COMPRESSION)

[hbase] branch branch-2 updated: HBASE-28145 When specifying the wrong BoolFilter type while creating a table in HBase shell, the log prompt will report an error. (#5460)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2 by this push:
 new e7237e2220d HBASE-28145 When specifying the wrong BoolFilter type 
while creating a table in HBase shell, the log prompt will report an error. 
(#5460)
e7237e2220d is described below

commit e7237e2220d9c6c0189c9bf0e5b7cf27b90e85b4
Author: ZhangIssac <58984599+zhangis...@users.noreply.github.com>
AuthorDate: Wed Oct 25 15:08:41 2023 +0800

HBASE-28145 When specifying the wrong BoolFilter type while creating a 
table in HBase shell, the log prompt will report an error. (#5460)

Co-authored-by: xiaozhang 
Signed-off-by: Duo Zhang 
(cherry picked from commit 13d46e7f8cd488d9d352baef4eacb0d8681d0295)
---
 hbase-shell/src/main/ruby/hbase/admin.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb 
b/hbase-shell/src/main/ruby/hbase/admin.rb
index a6dcb541c71..91e1e2b8472 100644
--- a/hbase-shell/src/main/ruby/hbase/admin.rb
+++ b/hbase-shell/src/main/ruby/hbase/admin.rb
@@ -1087,7 +1087,7 @@ module Hbase
 if 
org.apache.hadoop.hbase.regionserver.BloomType.constants.include?(bloomtype)
   
family.setBloomFilterType(org.apache.hadoop.hbase.regionserver.BloomType.valueOf(bloomtype))
 else
-  raise(ArgumentError, "BloomFilter type #{bloomtype} is not 
supported. Use one of " + 
org.apache.hadoop.hbase.regionserver.StoreFile::BloomType.constants.join(' '))
+  raise(ArgumentError, "BloomFilter type #{bloomtype} is not 
supported. Use one of " + 
org.apache.hadoop.hbase.regionserver.BloomType.constants.join(' '))
 end
   end
   if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::COMPRESSION)

[hbase] branch branch-2.5 updated: HBASE-28145 When specifying the wrong BoolFilter type while creating a table in HBase shell, the log prompt will report an error. (#5460)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.5 by this push:
 new 46cfaff1710 HBASE-28145 When specifying the wrong BoolFilter type 
while creating a table in HBase shell, the log prompt will report an error. 
(#5460)
46cfaff1710 is described below

commit 46cfaff1710b0c740c00dfaecd997f89cdee4761
Author: ZhangIssac <58984599+zhangis...@users.noreply.github.com>
AuthorDate: Wed Oct 25 15:08:41 2023 +0800

HBASE-28145 When specifying the wrong BoolFilter type while creating a 
table in HBase shell, the log prompt will report an error. (#5460)

Co-authored-by: xiaozhang 
Signed-off-by: Duo Zhang 
(cherry picked from commit 13d46e7f8cd488d9d352baef4eacb0d8681d0295)
---
 hbase-shell/src/main/ruby/hbase/admin.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb 
b/hbase-shell/src/main/ruby/hbase/admin.rb
index 42f25111c1a..031a77b4433 100644
--- a/hbase-shell/src/main/ruby/hbase/admin.rb
+++ b/hbase-shell/src/main/ruby/hbase/admin.rb
@@ -1087,7 +1087,7 @@ module Hbase
 if 
org.apache.hadoop.hbase.regionserver.BloomType.constants.include?(bloomtype)
   
family.setBloomFilterType(org.apache.hadoop.hbase.regionserver.BloomType.valueOf(bloomtype))
 else
-  raise(ArgumentError, "BloomFilter type #{bloomtype} is not 
supported. Use one of " + 
org.apache.hadoop.hbase.regionserver.StoreFile::BloomType.constants.join(' '))
+  raise(ArgumentError, "BloomFilter type #{bloomtype} is not 
supported. Use one of " + 
org.apache.hadoop.hbase.regionserver.BloomType.constants.join(' '))
 end
   end
   if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::COMPRESSION)

[hbase] branch branch-2.4 updated: HBASE-28145 When specifying the wrong BoolFilter type while creating a table in HBase shell, the log prompt will report an error. (#5460)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.4 by this push:
 new 88e9477f8b4 HBASE-28145 When specifying the wrong BoolFilter type 
while creating a table in HBase shell, the log prompt will report an error. 
(#5460)
88e9477f8b4 is described below

commit 88e9477f8b49df99552d1a0278b9edd0f47def1f
Author: ZhangIssac <58984599+zhangis...@users.noreply.github.com>
AuthorDate: Wed Oct 25 15:08:41 2023 +0800

HBASE-28145 When specifying the wrong BoolFilter type while creating a 
table in HBase shell, the log prompt will report an error. (#5460)

Co-authored-by: xiaozhang 
Signed-off-by: Duo Zhang 
(cherry picked from commit 13d46e7f8cd488d9d352baef4eacb0d8681d0295)
---
 hbase-shell/src/main/ruby/hbase/admin.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb 
b/hbase-shell/src/main/ruby/hbase/admin.rb
index a0bea3aa291..8e70a563d07 100644
--- a/hbase-shell/src/main/ruby/hbase/admin.rb
+++ b/hbase-shell/src/main/ruby/hbase/admin.rb
@@ -1039,7 +1039,7 @@ module Hbase
 if 
org.apache.hadoop.hbase.regionserver.BloomType.constants.include?(bloomtype)
   
family.setBloomFilterType(org.apache.hadoop.hbase.regionserver.BloomType.valueOf(bloomtype))
 else
-  raise(ArgumentError, "BloomFilter type #{bloomtype} is not 
supported. Use one of " + 
org.apache.hadoop.hbase.regionserver.StoreFile::BloomType.constants.join(' '))
+  raise(ArgumentError, "BloomFilter type #{bloomtype} is not 
supported. Use one of " + 
org.apache.hadoop.hbase.regionserver.BloomType.constants.join(' '))
 end
   end
   if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::COMPRESSION)

[hbase] branch master updated: HBASE-28145 When specifying the wrong BoolFilter type while creating a table in HBase shell, the log prompt will report an error. (#5460)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
 new 13d46e7f8cd HBASE-28145 When specifying the wrong BoolFilter type 
while creating a table in HBase shell, the log prompt will report an error. 
(#5460)
13d46e7f8cd is described below

commit 13d46e7f8cd488d9d352baef4eacb0d8681d0295
Author: ZhangIssac <58984599+zhangis...@users.noreply.github.com>
AuthorDate: Wed Oct 25 15:08:41 2023 +0800

HBASE-28145 When specifying the wrong BoolFilter type while creating a 
table in HBase shell, the log prompt will report an error. (#5460)

Co-authored-by: xiaozhang 
Signed-off-by: Duo Zhang 
---
 hbase-shell/src/main/ruby/hbase/admin.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb 
b/hbase-shell/src/main/ruby/hbase/admin.rb
index 53a8137fc0c..453b7ae1af6 100644
--- a/hbase-shell/src/main/ruby/hbase/admin.rb
+++ b/hbase-shell/src/main/ruby/hbase/admin.rb
@@ -1182,7 +1182,7 @@ module Hbase
 if 
org.apache.hadoop.hbase.regionserver.BloomType.constants.include?(bloomtype)
   
cfdb.setBloomFilterType(org.apache.hadoop.hbase.regionserver.BloomType.valueOf(bloomtype))
 else
-  raise(ArgumentError, "BloomFilter type #{bloomtype} is not 
supported. Use one of " + 
org.apache.hadoop.hbase.regionserver.StoreFile::BloomType.constants.join(' '))
+  raise(ArgumentError, "BloomFilter type #{bloomtype} is not 
supported. Use one of " + 
org.apache.hadoop.hbase.regionserver.BloomType.constants.join(' '))
 end
   end
   if arg.include?(ColumnFamilyDescriptorBuilder::COMPRESSION)

[hbase] branch branch-2.5 updated: HBASE-28146: Make ServerManager rsAdmins map thread safe (#5461)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.5 by this push:
 new 574fdc2d7a1 HBASE-28146: Make ServerManager rsAdmins map thread safe 
(#5461)
574fdc2d7a1 is described below

commit 574fdc2d7a1fdc1d15ba2c01e1355bc6464be10d
Author: Ray Mattingly 
AuthorDate: Mon Oct 23 05:16:14 2023 -0400

HBASE-28146: Make ServerManager rsAdmins map thread safe (#5461)

Co-authored-by: Ray Mattingly 
Signed-off-by: Duo Zhang 
Signed-off-by: Bryan Beaudreault 
(cherry picked from commit 1641a4abc13c7ad4da216a115107fc772d24d543)
---
 .../apache/hadoop/hbase/master/ServerManager.java  | 26 +-
 1 file changed, 6 insertions(+), 20 deletions(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index de08332f96e..0f058603c94 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -23,7 +23,6 @@ import java.io.IOException;
 import java.net.InetAddress;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -124,12 +123,6 @@ public class ServerManager {
   private final ConcurrentNavigableMap 
onlineServers =
 new ConcurrentSkipListMap<>();
 
-  /**
-   * Map of admin interfaces per registered regionserver; these interfaces we 
use to control
-   * regionservers out on the cluster
-   */
-  private final Map rsAdmins = new 
HashMap<>();
-
   /** List of region servers that should not get any more new regions. */
   private final ArrayList drainingServers = new ArrayList<>();
 
@@ -397,7 +390,6 @@ public class ServerManager {
   void recordNewServerWithLock(final ServerName serverName, final 
ServerMetrics sl) {
 LOG.info("Registering regionserver=" + serverName);
 this.onlineServers.put(serverName, sl);
-this.rsAdmins.remove(serverName);
   }
 
   public RegionStoreSequenceIds getLastFlushedSequenceId(byte[] 
encodedRegionName) {
@@ -599,7 +591,6 @@ public class ServerManager {
 LOG.trace("Expiration of {} but server not online", sn);
   }
 }
-this.rsAdmins.remove(sn);
   }
 
   /*
@@ -711,18 +702,13 @@ public class ServerManager {
* @throws RetriesExhaustedException wrapping a ConnectException if failed
*/
   public AdminService.BlockingInterface getRsAdmin(final ServerName sn) throws 
IOException {
-AdminService.BlockingInterface admin = this.rsAdmins.get(sn);
-if (admin == null) {
-  LOG.debug("New admin connection to " + sn.toString());
-  if (sn.equals(master.getServerName()) && master instanceof 
HRegionServer) {
-// A master is also a region server now, see HBASE-10569 for details
-admin = ((HRegionServer) master).getRSRpcServices();
-  } else {
-admin = this.connection.getAdmin(sn);
-  }
-  this.rsAdmins.put(sn, admin);
+LOG.debug("New admin connection to {}", sn);
+if (sn.equals(master.getServerName()) && master instanceof HRegionServer) {
+  // A master is also a region server now, see HBASE-10569 for details
+  return ((HRegionServer) master).getRSRpcServices();
+} else {
+  return this.connection.getAdmin(sn);
 }
-return admin;
   }
 
   /**

[hbase] branch branch-2.4 updated: HBASE-28146: Make ServerManager rsAdmins map thread safe (#5461)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.4 by this push:
 new dc5539ce678 HBASE-28146: Make ServerManager rsAdmins map thread safe 
(#5461)
dc5539ce678 is described below

commit dc5539ce6780e81a6859a82918785ad3388fd39d
Author: Ray Mattingly 
AuthorDate: Mon Oct 23 05:16:14 2023 -0400

HBASE-28146: Make ServerManager rsAdmins map thread safe (#5461)

Co-authored-by: Ray Mattingly 
Signed-off-by: Duo Zhang 
Signed-off-by: Bryan Beaudreault 
(cherry picked from commit 1641a4abc13c7ad4da216a115107fc772d24d543)
---
 .../apache/hadoop/hbase/master/ServerManager.java  | 27 ++
 1 file changed, 7 insertions(+), 20 deletions(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 98a12e69217..937789266b0 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -23,7 +23,6 @@ import java.io.IOException;
 import java.net.InetAddress;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -44,6 +43,7 @@ import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.YouAreDeadException;
 import org.apache.hadoop.hbase.client.ClusterConnection;
 import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RetriesExhaustedException;
 import org.apache.hadoop.hbase.ipc.HBaseRpcController;
 import org.apache.hadoop.hbase.ipc.RemoteWithExtrasException;
 import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
@@ -123,12 +123,6 @@ public class ServerManager {
   private final ConcurrentNavigableMap 
onlineServers =
 new ConcurrentSkipListMap<>();
 
-  /**
-   * Map of admin interfaces per registered regionserver; these interfaces we 
use to control
-   * regionservers out on the cluster
-   */
-  private final Map rsAdmins = new 
HashMap<>();
-
   /** List of region servers that should not get any more new regions. */
   private final ArrayList drainingServers = new ArrayList<>();
 
@@ -393,7 +387,6 @@ public class ServerManager {
   void recordNewServerWithLock(final ServerName serverName, final 
ServerMetrics sl) {
 LOG.info("Registering regionserver=" + serverName);
 this.onlineServers.put(serverName, sl);
-this.rsAdmins.remove(serverName);
   }
 
   public RegionStoreSequenceIds getLastFlushedSequenceId(byte[] 
encodedRegionName) {
@@ -595,7 +588,6 @@ public class ServerManager {
 LOG.trace("Expiration of {} but server not online", sn);
   }
 }
-this.rsAdmins.remove(sn);
   }
 
   /*
@@ -707,18 +699,13 @@ public class ServerManager {
* @throws RetriesExhaustedException wrapping a ConnectException if failed
*/
   public AdminService.BlockingInterface getRsAdmin(final ServerName sn) throws 
IOException {
-AdminService.BlockingInterface admin = this.rsAdmins.get(sn);
-if (admin == null) {
-  LOG.debug("New admin connection to " + sn.toString());
-  if (sn.equals(master.getServerName()) && master instanceof 
HRegionServer) {
-// A master is also a region server now, see HBASE-10569 for details
-admin = ((HRegionServer) master).getRSRpcServices();
-  } else {
-admin = this.connection.getAdmin(sn);
-  }
-  this.rsAdmins.put(sn, admin);
+LOG.debug("New admin connection to {}", sn);
+if (sn.equals(master.getServerName()) && master instanceof HRegionServer) {
+  // A master is also a region server now, see HBASE-10569 for details
+  return ((HRegionServer) master).getRSRpcServices();
+} else {
+  return this.connection.getAdmin(sn);
 }
-return admin;
   }
 
   /**

[hbase] branch branch-2 updated: HBASE-28146: Make ServerManager rsAdmins map thread safe (#5461)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2 by this push:
 new 1641a4abc13 HBASE-28146: Make ServerManager rsAdmins map thread safe 
(#5461)
1641a4abc13 is described below

commit 1641a4abc13c7ad4da216a115107fc772d24d543
Author: Ray Mattingly 
AuthorDate: Mon Oct 23 05:16:14 2023 -0400

HBASE-28146: Make ServerManager rsAdmins map thread safe (#5461)

Co-authored-by: Ray Mattingly 
Signed-off-by: Duo Zhang 
Signed-off-by: Bryan Beaudreault 
---
 .../apache/hadoop/hbase/master/ServerManager.java  | 26 +-
 1 file changed, 6 insertions(+), 20 deletions(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index ed37fd95444..196a1a582ed 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -23,7 +23,6 @@ import java.io.IOException;
 import java.net.InetAddress;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -124,12 +123,6 @@ public class ServerManager {
   private final ConcurrentNavigableMap 
onlineServers =
 new ConcurrentSkipListMap<>();
 
-  /**
-   * Map of admin interfaces per registered regionserver; these interfaces we 
use to control
-   * regionservers out on the cluster
-   */
-  private final Map rsAdmins = new 
HashMap<>();
-
   /** List of region servers that should not get any more new regions. */
   private final ArrayList drainingServers = new ArrayList<>();
 
@@ -402,7 +395,6 @@ public class ServerManager {
   void recordNewServerWithLock(final ServerName serverName, final 
ServerMetrics sl) {
 LOG.info("Registering regionserver=" + serverName);
 this.onlineServers.put(serverName, sl);
-this.rsAdmins.remove(serverName);
   }
 
   public RegionStoreSequenceIds getLastFlushedSequenceId(byte[] 
encodedRegionName) {
@@ -604,7 +596,6 @@ public class ServerManager {
 LOG.trace("Expiration of {} but server not online", sn);
   }
 }
-this.rsAdmins.remove(sn);
   }
 
   /*
@@ -716,18 +707,13 @@ public class ServerManager {
* @throws RetriesExhaustedException wrapping a ConnectException if failed
*/
   public AdminService.BlockingInterface getRsAdmin(final ServerName sn) throws 
IOException {
-AdminService.BlockingInterface admin = this.rsAdmins.get(sn);
-if (admin == null) {
-  LOG.debug("New admin connection to " + sn.toString());
-  if (sn.equals(master.getServerName()) && master instanceof 
HRegionServer) {
-// A master is also a region server now, see HBASE-10569 for details
-admin = ((HRegionServer) master).getRSRpcServices();
-  } else {
-admin = this.connection.getAdmin(sn);
-  }
-  this.rsAdmins.put(sn, admin);
+LOG.debug("New admin connection to {}", sn);
+if (sn.equals(master.getServerName()) && master instanceof HRegionServer) {
+  // A master is also a region server now, see HBASE-10569 for details
+  return ((HRegionServer) master).getRSRpcServices();
+} else {
+  return this.connection.getAdmin(sn);
 }
-return admin;
   }
 
   /**

[hbase] branch branch-3 updated: HBASE-28114 Add more comments to explain why replication log queue could never be empty for normal replication queue (#5443)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-3 by this push:
 new d7b6e8b2f49 HBASE-28114 Add more comments to explain why replication 
log queue could never be empty for normal replication queue (#5443)
d7b6e8b2f49 is described below

commit d7b6e8b2f49e03696777d2f37c1776d0dd5468d8
Author: Duo Zhang 
AuthorDate: Fri Oct 20 22:22:16 2023 +0800

HBASE-28114 Add more comments to explain why replication log queue could 
never be empty for normal replication queue (#5443)

Also add a retry logic to make the code more robust

Signed-off-by: Xiaolin Ha 
(cherry picked from commit 4429de48bace58f7581a3ad568c19531a1697071)
---
 .../replication/regionserver/WALEntryStream.java   | 34 ++
 1 file changed, 34 insertions(+)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/WALEntryStream.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/WALEntryStream.java
index c6268674c5b..d1f85774a63 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/WALEntryStream.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/WALEntryStream.java
@@ -334,11 +334,35 @@ class WALEntryStream implements Closeable {
 boolean beingWritten = pair.getSecond();
 LOG.trace("Reading WAL {}; result={}, currently open for write={}", 
this.currentPath, state,
   beingWritten);
+// The below implementation needs to make sure that when beingWritten == 
true, we should not
+// dequeue the current WAL file in logQueue.
 switch (state) {
   case NORMAL:
 // everything is fine, just return
 return HasNext.YES;
   case EOF_WITH_TRAILER:
+// in readNextEntryAndRecordReaderPosition, we will acquire 
rollWriteLock, and we can only
+// schedule a close writer task, in which we will write trailer, under 
the rollWriteLock, so
+// typically if beingWritten == true, we should not reach here, as we 
need to reopen the
+// reader after writing the trailer. The only possible way to reach 
here while beingWritten
+// == true is due to the inflightWALClosures logic in AbstractFSWAL, 
as if the writer is
+// still in this map, we will consider it as beingWritten, but 
actually, here we could make
+// sure that the new WAL file has already been enqueued into the 
logQueue, so here dequeuing
+// the current log file is safe.
+if (beingWritten && logQueue.getQueue(walGroupId).size() <= 1) {
+  // As explained above, if we implement everything correctly, we 
should not arrive here.
+  // But anyway, even if we reach here due to some code changes in the 
future, reading
+  // the file again can make sure that we will not accidentally 
consider the queue as
+  // finished, and since there is a trailer, we will soon consider the 
file as finished
+  // and move on.
+  LOG.warn(
+"We have reached the trailer while reading the file '{}' which is 
currently"
+  + " beingWritten, but it is the last file in log queue {}. This 
should not happen"
+  + " typically, try to read again so we will not miss anything",
+currentPath, walGroupId);
+  return HasNext.RETRY;
+}
+assert !beingWritten || logQueue.getQueue(walGroupId).size() > 1;
 // we have reached the trailer, which means this WAL file has been 
closed cleanly and we
 // have finished reading it successfully, just move to the next WAL 
file and let the upper
 // layer start reading the next WAL file
@@ -436,6 +460,16 @@ class WALEntryStream implements Closeable {
* Returns whether the file is opened for writing.
*/
   private Pair 
readNextEntryAndRecordReaderPosition() {
+// we must call this before actually reading from the reader, as this 
method will acquire the
+// rollWriteLock. This is very important, as we will enqueue the new WAL 
file in postLogRoll,
+// and before this happens, we could have already finished closing the 
previous WAL file. If we
+// do not acquire the rollWriteLock and return whether the current file is 
being written to, we
+// may finish reading the previous WAL file and start to read the next 
one, before it is
+// enqueued into the logQueue, thus lead to an empty logQueue and make the 
shipper think the
+// queue is already ended and quit. See HBASE-28114 and related issues for 
more details.
+// in the future, if we want to optimize the logic here, for example, do 
not call this method
+// every time, or do not acquire rollWriteLock in the implementation of 
this met

[hbase] branch branch-2 updated: HBASE-28114 Add more comments to explain why replication log queue could never be empty for normal replication queue (#5443)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2 by this push:
 new 2951e564745 HBASE-28114 Add more comments to explain why replication 
log queue could never be empty for normal replication queue (#5443)
2951e564745 is described below

commit 2951e564745a7deafc6b723077ba85f36b83ab46
Author: Duo Zhang 
AuthorDate: Fri Oct 20 22:22:16 2023 +0800

HBASE-28114 Add more comments to explain why replication log queue could 
never be empty for normal replication queue (#5443)

Also add a retry logic to make the code more robust

Signed-off-by: Xiaolin Ha 
(cherry picked from commit 4429de48bace58f7581a3ad568c19531a1697071)
---
 .../replication/regionserver/WALEntryStream.java   | 34 ++
 1 file changed, 34 insertions(+)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/WALEntryStream.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/WALEntryStream.java
index 22bf05b3741..8953d18a271 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/WALEntryStream.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/WALEntryStream.java
@@ -340,11 +340,35 @@ class WALEntryStream implements Closeable {
 boolean beingWritten = pair.getSecond();
 LOG.trace("Reading WAL {}; result={}, currently open for write={}", 
this.currentPath, state,
   beingWritten);
+// The below implementation needs to make sure that when beingWritten == 
true, we should not
+// dequeue the current WAL file in logQueue.
 switch (state) {
   case NORMAL:
 // everything is fine, just return
 return HasNext.YES;
   case EOF_WITH_TRAILER:
+// in readNextEntryAndRecordReaderPosition, we will acquire 
rollWriteLock, and we can only
+// schedule a close writer task, in which we will write trailer, under 
the rollWriteLock, so
+// typically if beingWritten == true, we should not reach here, as we 
need to reopen the
+// reader after writing the trailer. The only possible way to reach 
here while beingWritten
+// == true is due to the inflightWALClosures logic in AbstractFSWAL, 
as if the writer is
+// still in this map, we will consider it as beingWritten, but 
actually, here we could make
+// sure that the new WAL file has already been enqueued into the 
logQueue, so here dequeuing
+// the current log file is safe.
+if (beingWritten && logQueue.getQueue(walGroupId).size() <= 1) {
+  // As explained above, if we implement everything correctly, we 
should not arrive here.
+  // But anyway, even if we reach here due to some code changes in the 
future, reading
+  // the file again can make sure that we will not accidentally 
consider the queue as
+  // finished, and since there is a trailer, we will soon consider the 
file as finished
+  // and move on.
+  LOG.warn(
+"We have reached the trailer while reading the file '{}' which is 
currently"
+  + " beingWritten, but it is the last file in log queue {}. This 
should not happen"
+  + " typically, try to read again so we will not miss anything",
+currentPath, walGroupId);
+  return HasNext.RETRY;
+}
+assert !beingWritten || logQueue.getQueue(walGroupId).size() > 1;
 // we have reached the trailer, which means this WAL file has been 
closed cleanly and we
 // have finished reading it successfully, just move to the next WAL 
file and let the upper
 // layer start reading the next WAL file
@@ -442,6 +466,16 @@ class WALEntryStream implements Closeable {
* Returns whether the file is opened for writing.
*/
   private Pair 
readNextEntryAndRecordReaderPosition() {
+// we must call this before actually reading from the reader, as this 
method will acquire the
+// rollWriteLock. This is very important, as we will enqueue the new WAL 
file in postLogRoll,
+// and before this happens, we could have already finished closing the 
previous WAL file. If we
+// do not acquire the rollWriteLock and return whether the current file is 
being written to, we
+// may finish reading the previous WAL file and start to read the next 
one, before it is
+// enqueued into the logQueue, thus lead to an empty logQueue and make the 
shipper think the
+// queue is already ended and quit. See HBASE-28114 and related issues for 
more details.
+// in the future, if we want to optimize the logic here, for example, do 
not call this method
+// every time, or do not acquire rollWriteLock in the implementation of 
this met

[hbase] branch master updated: HBASE-28114 Add more comments to explain why replication log queue could never be empty for normal replication queue (#5443)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
 new 4429de48bac HBASE-28114 Add more comments to explain why replication 
log queue could never be empty for normal replication queue (#5443)
4429de48bac is described below

commit 4429de48bace58f7581a3ad568c19531a1697071
Author: Duo Zhang 
AuthorDate: Fri Oct 20 22:22:16 2023 +0800

HBASE-28114 Add more comments to explain why replication log queue could 
never be empty for normal replication queue (#5443)

Also add a retry logic to make the code more robust

Signed-off-by: Xiaolin Ha 
---
 .../replication/regionserver/WALEntryStream.java   | 34 ++
 1 file changed, 34 insertions(+)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/WALEntryStream.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/WALEntryStream.java
index c6268674c5b..d1f85774a63 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/WALEntryStream.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/WALEntryStream.java
@@ -334,11 +334,35 @@ class WALEntryStream implements Closeable {
 boolean beingWritten = pair.getSecond();
 LOG.trace("Reading WAL {}; result={}, currently open for write={}", 
this.currentPath, state,
   beingWritten);
+// The below implementation needs to make sure that when beingWritten == 
true, we should not
+// dequeue the current WAL file in logQueue.
 switch (state) {
   case NORMAL:
 // everything is fine, just return
 return HasNext.YES;
   case EOF_WITH_TRAILER:
+// in readNextEntryAndRecordReaderPosition, we will acquire 
rollWriteLock, and we can only
+// schedule a close writer task, in which we will write trailer, under 
the rollWriteLock, so
+// typically if beingWritten == true, we should not reach here, as we 
need to reopen the
+// reader after writing the trailer. The only possible way to reach 
here while beingWritten
+// == true is due to the inflightWALClosures logic in AbstractFSWAL, 
as if the writer is
+// still in this map, we will consider it as beingWritten, but 
actually, here we could make
+// sure that the new WAL file has already been enqueued into the 
logQueue, so here dequeuing
+// the current log file is safe.
+if (beingWritten && logQueue.getQueue(walGroupId).size() <= 1) {
+  // As explained above, if we implement everything correctly, we 
should not arrive here.
+  // But anyway, even if we reach here due to some code changes in the 
future, reading
+  // the file again can make sure that we will not accidentally 
consider the queue as
+  // finished, and since there is a trailer, we will soon consider the 
file as finished
+  // and move on.
+  LOG.warn(
+"We have reached the trailer while reading the file '{}' which is 
currently"
+  + " beingWritten, but it is the last file in log queue {}. This 
should not happen"
+  + " typically, try to read again so we will not miss anything",
+currentPath, walGroupId);
+  return HasNext.RETRY;
+}
+assert !beingWritten || logQueue.getQueue(walGroupId).size() > 1;
 // we have reached the trailer, which means this WAL file has been 
closed cleanly and we
 // have finished reading it successfully, just move to the next WAL 
file and let the upper
 // layer start reading the next WAL file
@@ -436,6 +460,16 @@ class WALEntryStream implements Closeable {
* Returns whether the file is opened for writing.
*/
   private Pair 
readNextEntryAndRecordReaderPosition() {
+// we must call this before actually reading from the reader, as this 
method will acquire the
+// rollWriteLock. This is very important, as we will enqueue the new WAL 
file in postLogRoll,
+// and before this happens, we could have already finished closing the 
previous WAL file. If we
+// do not acquire the rollWriteLock and return whether the current file is 
being written to, we
+// may finish reading the previous WAL file and start to read the next 
one, before it is
+// enqueued into the logQueue, thus lead to an empty logQueue and make the 
shipper think the
+// queue is already ended and quit. See HBASE-28114 and related issues for 
more details.
+// in the future, if we want to optimize the logic here, for example, do 
not call this method
+// every time, or do not acquire rollWriteLock in the implementation of 
this method, we need to
+// carefully review the

[hbase] branch branch-3 updated: HBASE-28155 RecoveredReplicationSource quit when there are still unfinished groups (#5466)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-3 by this push:
 new 0f82a447585 HBASE-28155 RecoveredReplicationSource quit when there are 
still unfinished groups (#5466)
0f82a447585 is described below

commit 0f82a447585b33d93dbc00a6f5da3dcdd1a5c571
Author: Duo Zhang 
AuthorDate: Fri Oct 20 11:58:28 2023 +0800

HBASE-28155 RecoveredReplicationSource quit when there are still unfinished 
groups (#5466)

Signed-off-by: Guanghao Zhang 
(cherry picked from commit dde504ce489fd3fd55166a872768a077400ba2ab)
---
 .../regionserver/RecoveredReplicationSource.java   | 16 
 .../regionserver/ReplicationSource.java| 29 --
 2 files changed, 37 insertions(+), 8 deletions(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
index e9062472221..e47df36e3aa 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
@@ -26,6 +26,22 @@ import org.apache.yetus.audience.InterfaceAudience;
 @InterfaceAudience.Private
 public class RecoveredReplicationSource extends ReplicationSource {
 
+  @Override
+  protected void startShippers() {
+for (String walGroupId : logQueue.getQueues().keySet()) {
+  workerThreads.put(walGroupId, createNewShipper(walGroupId));
+}
+// start shippers after initializing the workerThreads, as in the below 
postFinish logic, if
+// workerThreads is empty, we will mark the RecoveredReplicationSource as 
finished. So if we
+// start the worker on the fly, it is possible that a shipper has already 
finished its work and
+// called postFinish, and find out the workerThreads is empty and then 
mark the
+// RecoveredReplicationSource as finish, while the next shipper has not 
been added to
+// workerThreads yet. See HBASE-28155 for more details.
+for (ReplicationSourceShipper shipper : workerThreads.values()) {
+  startShipper(shipper);
+}
+  }
+
   @Override
   protected RecoveredReplicationSourceShipper createNewShipper(String 
walGroupId,
 ReplicationSourceWALReader walReader) {
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 4c864e5e450..094fa4aaa78 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -360,6 +360,19 @@ public class ReplicationSource implements 
ReplicationSourceInterface {
 }
   }
 
+  protected final ReplicationSourceShipper createNewShipper(String walGroupId) 
{
+ReplicationSourceWALReader walReader =
+  createNewWALReader(walGroupId, getStartOffset(walGroupId));
+ReplicationSourceShipper worker = createNewShipper(walGroupId, walReader);
+Threads.setDaemonThreadRunning(walReader, Thread.currentThread().getName()
+  + ".replicationSource.wal-reader." + walGroupId + "," + queueId, 
this::retryRefreshing);
+return worker;
+  }
+
+  protected final void startShipper(ReplicationSourceShipper worker) {
+worker.startup(this::retryRefreshing);
+  }
+
   private void tryStartNewShipper(String walGroupId) {
 workerThreads.compute(walGroupId, (key, value) -> {
   if (value != null) {
@@ -367,12 +380,8 @@ public class ReplicationSource implements 
ReplicationSourceInterface {
 return value;
   } else {
 LOG.debug("{} starting shipping worker for walGroupId={}", 
logPeerId(), walGroupId);
-ReplicationSourceWALReader walReader =
-  createNewWALReader(walGroupId, getStartOffset(walGroupId));
-ReplicationSourceShipper worker = createNewShipper(walGroupId, 
walReader);
-Threads.setDaemonThreadRunning(walReader, 
Thread.currentThread().getName()
-  + ".replicationSource.wal-reader." + walGroupId + "," + queueId, 
this::retryRefreshing);
-worker.startup(this::retryRefreshing);
+ReplicationSourceShipper worker = createNewShipper(walGroupId);
+startShipper(worker);
 return worker;
   }
 });
@@ -522,7 +531,7 @@ public class ReplicationSource implements 
ReplicationSourceInterface {
* @param sleepMultiplier by how many times the default sleeping time is 
augmented
* @return True if sleepMultiplier is  
maxRetriesMultiplier
 

[hbase] branch master updated: HBASE-28155 RecoveredReplicationSource quit when there are still unfinished groups (#5466)

[zhangduo]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
 new dde504ce489 HBASE-28155 RecoveredReplicationSource quit when there are 
still unfinished groups (#5466)
dde504ce489 is described below

commit dde504ce489fd3fd55166a872768a077400ba2ab
Author: Duo Zhang 
AuthorDate: Fri Oct 20 11:58:28 2023 +0800

HBASE-28155 RecoveredReplicationSource quit when there are still unfinished 
groups (#5466)

Signed-off-by: Guanghao Zhang 
---
 .../regionserver/RecoveredReplicationSource.java   | 16 
 .../regionserver/ReplicationSource.java| 29 --
 2 files changed, 37 insertions(+), 8 deletions(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
index e9062472221..e47df36e3aa 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
@@ -26,6 +26,22 @@ import org.apache.yetus.audience.InterfaceAudience;
 @InterfaceAudience.Private
 public class RecoveredReplicationSource extends ReplicationSource {
 
+  @Override
+  protected void startShippers() {
+for (String walGroupId : logQueue.getQueues().keySet()) {
+  workerThreads.put(walGroupId, createNewShipper(walGroupId));
+}
+// start shippers after initializing the workerThreads, as in the below 
postFinish logic, if
+// workerThreads is empty, we will mark the RecoveredReplicationSource as 
finished. So if we
+// start the worker on the fly, it is possible that a shipper has already 
finished its work and
+// called postFinish, and find out the workerThreads is empty and then 
mark the
+// RecoveredReplicationSource as finish, while the next shipper has not 
been added to
+// workerThreads yet. See HBASE-28155 for more details.
+for (ReplicationSourceShipper shipper : workerThreads.values()) {
+  startShipper(shipper);
+}
+  }
+
   @Override
   protected RecoveredReplicationSourceShipper createNewShipper(String 
walGroupId,
 ReplicationSourceWALReader walReader) {
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 4c864e5e450..094fa4aaa78 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -360,6 +360,19 @@ public class ReplicationSource implements 
ReplicationSourceInterface {
 }
   }
 
+  protected final ReplicationSourceShipper createNewShipper(String walGroupId) 
{
+ReplicationSourceWALReader walReader =
+  createNewWALReader(walGroupId, getStartOffset(walGroupId));
+ReplicationSourceShipper worker = createNewShipper(walGroupId, walReader);
+Threads.setDaemonThreadRunning(walReader, Thread.currentThread().getName()
+  + ".replicationSource.wal-reader." + walGroupId + "," + queueId, 
this::retryRefreshing);
+return worker;
+  }
+
+  protected final void startShipper(ReplicationSourceShipper worker) {
+worker.startup(this::retryRefreshing);
+  }
+
   private void tryStartNewShipper(String walGroupId) {
 workerThreads.compute(walGroupId, (key, value) -> {
   if (value != null) {
@@ -367,12 +380,8 @@ public class ReplicationSource implements 
ReplicationSourceInterface {
 return value;
   } else {
 LOG.debug("{} starting shipping worker for walGroupId={}", 
logPeerId(), walGroupId);
-ReplicationSourceWALReader walReader =
-  createNewWALReader(walGroupId, getStartOffset(walGroupId));
-ReplicationSourceShipper worker = createNewShipper(walGroupId, 
walReader);
-Threads.setDaemonThreadRunning(walReader, 
Thread.currentThread().getName()
-  + ".replicationSource.wal-reader." + walGroupId + "," + queueId, 
this::retryRefreshing);
-worker.startup(this::retryRefreshing);
+ReplicationSourceShipper worker = createNewShipper(walGroupId);
+startShipper(worker);
 return worker;
   }
 });
@@ -522,7 +531,7 @@ public class ReplicationSource implements 
ReplicationSourceInterface {
* @param sleepMultiplier by how many times the default sleeping time is 
augmented
* @return True if sleepMultiplier is  
maxRetriesMultiplier
*/
-  protected boolean sleepForRetries(String msg, int sleepMultiplier)