svn commit: r56456 [3/4] - /dev/hbase/2.4.14RC0/
Added: dev/hbase/2.4.14RC0/RELEASENOTES.md == --- dev/hbase/2.4.14RC0/RELEASENOTES.md (added) +++ dev/hbase/2.4.14RC0/RELEASENOTES.md Mon Aug 22 21:59:28 2022 @@ -0,0 +1,20173 @@ +# RELEASENOTES + + +# HBASE 2.4.14 Release Notes + +These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements. + + +--- + +* [HBASE-27129](https://issues.apache.org/jira/browse/HBASE-27129) | *Major* | **Add a config that allows us to configure region-level storage policies** + +Add a 'hbase.hregion.block.storage.policy' so you can config storage policy at region level. This is useful when you want to control the storage policy for the directories other than CF directories, such as .splits, .recovered.edits, etc. + + +--- + +* [HBASE-27089](https://issues.apache.org/jira/browse/HBASE-27089) | *Minor* | **Add âcommons.crypto.stream.buffer.sizeâ configuration** + +Add a 'commons.crypto.stream.buffer.size' config for setting the buffer size when doing AES crypto for RPC. + + +--- + +* [HBASE-27281](https://issues.apache.org/jira/browse/HBASE-27281) | *Critical* | **Add default implementation for Connection$getClusterId** + +Adds a default null implementation for Connection$getClusterId. Downstream applications should implement this method. + + +--- + +* [HBASE-27251](https://issues.apache.org/jira/browse/HBASE-27251) | *Critical* | **Rolling back from 2.5.0-SNAPSHOT to 2.4.13 fails due to \`File does not exist: /hbase/MasterData/data/master/store/.initialized/.regioninfo\`** + +Before upgrading HBase cluster to 2.5.0, it is strongly recommended to upgrade the cluster to HBase-2.4.14 or later first. This is to guarantee a smooth rollback in case. Otherwise, it may run into HBASE-27251. + + +--- + +* [HBASE-27225](https://issues.apache.org/jira/browse/HBASE-27225) | *Major* | **Add BucketAllocator bucket size statistic logging** + +When bucket cache is enabled, setting loglevel to DEBUG for the org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator logger will result in a periodic summary of bucket usage for the bucket cache. This log output can give insight into block distribution of the cluster, where bucket cache memory is being allocated, and how much waste/fragmentation there is in the bucket cache. See Javadoc in BucketAllocate for more explanation of the logging. + + +--- + +* [HBASE-27204](https://issues.apache.org/jira/browse/HBASE-27204) | *Critical* | **BlockingRpcClient will hang for 20 seconds when SASL is enabled after finishing negotiation** + +When Kerberos authentication succeeds, on the server side, after receiving the final SASL token from the client, we simply wait for the client to continue by sending the connection header. After HBASE-24579, on the client side, an additional readStatus() was added, which assumed that after negotiation has completed a status code will be sent. However when authentication has succeeded the server will not send one. As a result the client would hang and only throw an exception when the configured read timeout is reached, which is 20 seconds by default. This was especially noticeable when using BlockingRpcClient as the client implementation. HBASE-24579 was reverted to correct this issue. + + +--- + +* [HBASE-27219](https://issues.apache.org/jira/browse/HBASE-27219) | *Minor* | **Change JONI encoding in RegexStringComparator** + +In RegexStringComparator an infinite loop can occur if an invalid UTF8 is encountered. We now use joni's NonStrictUTF8Encoding instead of UTF8Encoding to avoid the issue. + + +--- + +* [HBASE-27048](https://issues.apache.org/jira/browse/HBASE-27048) | *Major* | **Server side scanner time limit should account for time in queue** + +Server will now account for queue time when determining how long a scanner can run before heartbeat should be returned. This should help avoid timeouts when server is overloaded. + + +--- + +* [HBASE-27060](https://issues.apache.org/jira/browse/HBASE-27060) | *Major* | **Allow sharing connections between AggregationClient instances** + +Added a new no-arg constructor for AggregationClient which allows lightweight usage of just the methods accepting a Table. Also added a new constructor which takes a Connection, allowing the user to pass their own externally managed connection which will be used when methods accepting a TableName are used. + + +--- + +* [HBASE-26790](https://issues.apache.org/jira/browse/HBASE-26790) | *Major* | **getAllRegionLocations can cache locations with null hostname** + +getAllRegionLocations will no longer add locations to the MetaCache which have no server name. Region locations without a server name can occur for brief periods during region splits and merges, but the MetaCache relies on every cached location having a server name. + + +--- + +* [HBASE-26945](https://issues.apache.org/jira/browse/HBASE-26945) |
svn commit: r56456 [1/4] - /dev/hbase/2.4.14RC0/
Author: huaxiangsun Date: Mon Aug 22 21:59:28 2022 New Revision: 56456 Log: Apache hbase 2.4.14RC0 Added: dev/hbase/2.4.14RC0/ dev/hbase/2.4.14RC0/CHANGES.md dev/hbase/2.4.14RC0/RELEASENOTES.md dev/hbase/2.4.14RC0/api_compare_2.4.13_to_2.4.14RC0.html dev/hbase/2.4.14RC0/hbase-2.4.14-bin.tar.gz (with props) dev/hbase/2.4.14RC0/hbase-2.4.14-bin.tar.gz.asc dev/hbase/2.4.14RC0/hbase-2.4.14-bin.tar.gz.sha512 dev/hbase/2.4.14RC0/hbase-2.4.14-client-bin.tar.gz (with props) dev/hbase/2.4.14RC0/hbase-2.4.14-client-bin.tar.gz.asc dev/hbase/2.4.14RC0/hbase-2.4.14-client-bin.tar.gz.sha512 dev/hbase/2.4.14RC0/hbase-2.4.14-src.tar.gz (with props) dev/hbase/2.4.14RC0/hbase-2.4.14-src.tar.gz.asc dev/hbase/2.4.14RC0/hbase-2.4.14-src.tar.gz.sha512
svn commit: r56456 [2/4] - /dev/hbase/2.4.14RC0/
Added: dev/hbase/2.4.14RC0/CHANGES.md == --- dev/hbase/2.4.14RC0/CHANGES.md (added) +++ dev/hbase/2.4.14RC0/CHANGES.md Mon Aug 22 21:59:28 2022 @@ -0,0 +1,1389 @@ +# HBASE Changelog + + +## Release 2.4.14 - Unreleased (as of 2022-08-22) + + + +### NEW FEATURES: + +| JIRA | Summary | Priority | Component | +|: |: | :--- |: | +| [HBASE-27129](https://issues.apache.org/jira/browse/HBASE-27129) | Add a config that allows us to configure region-level storage policies | Major | regionserver | + + +### IMPROVEMENTS: + +| JIRA | Summary | Priority | Component | +|: |: | :--- |: | +| [HBASE-27089](https://issues.apache.org/jira/browse/HBASE-27089) | Add âcommons.crypto.stream.buffer.sizeâ configuration | Minor | io | +| [HBASE-27268](https://issues.apache.org/jira/browse/HBASE-27268) | In trace log mode, the client does not print callId/startTime and the server does not print receiveTime | Minor | logging | +| [HBASE-27296](https://issues.apache.org/jira/browse/HBASE-27296) | Some Cell's implementation of toString() such as IndividualBytesFieldCell prints out value and tags which is too verbose | Minor | logging | +| [HBASE-27273](https://issues.apache.org/jira/browse/HBASE-27273) | Should stop autoRead and skip all the bytes when rpc request too big | Major | IPC/RPC | +| [HBASE-27257](https://issues.apache.org/jira/browse/HBASE-27257) | Remove unnecessary usage of CachedBlocksByFile from RS UI | Major | . | +| [HBASE-27225](https://issues.apache.org/jira/browse/HBASE-27225) | Add BucketAllocator bucket size statistic logging | Major | . | +| [HBASE-27208](https://issues.apache.org/jira/browse/HBASE-27208) | Use spotless to purge the missing summary warnings from error prone | Major | pom | +| [HBASE-27048](https://issues.apache.org/jira/browse/HBASE-27048) | Server side scanner time limit should account for time in queue | Major | . | +| [HBASE-27188](https://issues.apache.org/jira/browse/HBASE-27188) | Report maxStoreFileCount in jmx | Minor | . | +| [HBASE-27186](https://issues.apache.org/jira/browse/HBASE-27186) | Report block cache size metrics separately for L1 and L2 | Minor | . | +| [HBASE-26218](https://issues.apache.org/jira/browse/HBASE-26218) | Better logging in CanaryTool | Minor | canary | +| [HBASE-27060](https://issues.apache.org/jira/browse/HBASE-27060) | Allow sharing connections between AggregationClient instances | Major | . | +| [HBASE-27146](https://issues.apache.org/jira/browse/HBASE-27146) | Avoid CellUtil.cloneRow in MetaCellComparator | Major | meta, Offheaping, Performance | +| [HBASE-26945](https://issues.apache.org/jira/browse/HBASE-26945) | Quotas causes too much load on meta for large clusters | Minor | . | +| [HBASE-27125](https://issues.apache.org/jira/browse/HBASE-27125) | The batch size of cleaning expired mob files should have an upper bound | Minor | mob | + + +### BUG FIXES: + +| JIRA | Summary | Priority | Component | +|: |: | :--- |: | +| [HBASE-27292](https://issues.apache.org/jira/browse/HBASE-27292) | Fix build failure against Hadoop 3.3.4 due to added dependency on okhttp | Major | build, hadoop3, pom | +| [HBASE-27244](https://issues.apache.org/jira/browse/HBASE-27244) | bin/hbase still use slf4j-log4j while reload4j in place | Major | shell | +| [HBASE-27275](https://issues.apache.org/jira/browse/HBASE-27275) | graceful\_stop.sh unable to restore the balance state | Blocker | regionserver | +| [HBASE-27282](https://issues.apache.org/jira/browse/HBASE-27282) | CME in AuthManager causes region server crash | Major | acl | +| [HBASE-26775](https://issues.apache.org/jira/browse/HBASE-26775) | TestProcedureSchedulerConcurrency fails in pre commit | Major | proc-v2, test | +| [HBASE-27269](https://issues.apache.org/jira/browse/HBASE-27269) | The implementation of TestReplicationStatus.waitOnMetricsReport is incorrect | Major | Replication, test | +| [HBASE-27271](https://issues.apache.org/jira/browse/HBASE-27271) | BufferCallBeforeInitHandler should ignore the flush request | Major | IPC/RPC | +| [HBASE-27251](https://issues.apache.org/jira/browse/HBASE-27251) | Rolling back from 2.5.0-SNAPSHOT to 2.4.13 fails due to \`File does not exist: /hbase/MasterData/data/master/store/.initialized/.regioninfo\` | Critical | master | +| [HBASE-27087](https://issues.apache.org/jira/browse/HBASE-27087) | TestQuotaThrottle times out | Major | test | +| [HBASE-27239](https://issues.apache.org/jira/browse/HBASE-27239) | Upgrade reload4j due to XXE vulnerability | Major | . | +| [HBASE-27204](https://issues.apache.org/jira/browse/HBASE-27204) | BlockingRpcClient will hang for 20 seconds when SASL is enabled after finishing negotiation | Critical | rpc, sasl, security | +| [HBASE-27219](https://issues.apache.org/jira/browse/HBASE-27219) | Change JONI encoding in RegexStringComparator | Minor | Filters | +|
svn commit: r56456 [4/4] - /dev/hbase/2.4.14RC0/
Added: dev/hbase/2.4.14RC0/api_compare_2.4.13_to_2.4.14RC0.html
==
--- dev/hbase/2.4.14RC0/api_compare_2.4.13_to_2.4.14RC0.html (added)
+++ dev/hbase/2.4.14RC0/api_compare_2.4.13_to_2.4.14RC0.html Mon Aug 22
21:59:28 2022
@@ -0,0 +1,541 @@
+
+
+http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd;>
+http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
+
+
+
+
+
+hbase: rel/2.4.13 to 2.4.14RC0 compatibility report
+
+body {
+font-family:Arial, sans-serif;
+background-color:White;
+color:Black;
+}
+hr {
+color:Black;
+background-color:Black;
+height:1px;
+border:0;
+}
+h1 {
+margin-bottom:0px;
+padding-bottom:0px;
+font-size:1.625em;
+}
+h2 {
+margin-bottom:0px;
+padding-bottom:0px;
+font-size:1.25em;
+white-space:nowrap;
+}
+div.symbols {
+color:#003E69;
+}
+div.symbols i {
+color:Brown;
+}
+span.section {
+font-weight:bold;
+cursor:pointer;
+color:#003E69;
+white-space:nowrap;
+margin-left:0.3125em;
+}
+span:hover.section {
+color:#336699;
+}
+span.sect_aff {
+cursor:pointer;
+padding-left:1.55em;
+font-size:0.875em;
+color:#cc3300;
+}
+span.ext {
+font-weight:normal;
+}
+span.jar {
+color:#cc3300;
+font-size:0.875em;
+font-weight:bold;
+}
+div.jar_list {
+padding-left:0.4em;
+font-size:0.94em;
+}
+span.pkg_t {
+color:#408080;
+font-size:0.875em;
+}
+span.pkg {
+color:#408080;
+font-size:0.875em;
+font-weight:bold;
+}
+span.cname {
+color:Green;
+font-size:0.875em;
+font-weight:bold;
+}
+span.iname_b {
+font-weight:bold;
+}
+span.iname_a {
+color:#33;
+font-weight:bold;
+font-size:0.94em;
+}
+span.sym_p {
+font-weight:normal;
+white-space:normal;
+}
+span.sym_pd {
+white-space:normal;
+}
+span.sym_p span, span.sym_pd span {
+white-space:nowrap;
+}
+span.attr {
+color:Black;
+font-weight:normal;
+}
+span.deprecated {
+color:Red;
+font-weight:bold;
+font-family:Monaco, monospace;
+}
+div.affect {
+padding-left:1em;
+padding-bottom:10px;
+font-size:0.87em;
+font-style:italic;
+line-height:0.9em;
+}
+div.affected {
+padding-left:2em;
+padding-top:10px;
+}
+table.ptable {
+border-collapse:collapse;
+border:1px outset black;
+margin-left:0.95em;
+margin-top:3px;
+margin-bottom:3px;
+width:56.25em;
+}
+table.ptable td {
+border:1px solid Gray;
+padding:3px;
+font-size:0.875em;
+text-align:left;
+vertical-align:top;
+max-width:28em;
+word-wrap:break-word;
+}
+table.ptable th {
+background-color:#ee;
+font-weight:bold;
+color:#33;
+font-family:Verdana, Arial;
+font-size:0.875em;
+border:1px solid Gray;
+text-align:center;
+vertical-align:top;
+white-space:nowrap;
+padding:3px;
+}
+table.summary {
+border-collapse:collapse;
+border:1px outset black;
+}
+table.summary th {
+background-color:#ee;
+font-weight:normal;
+text-align:left;
+font-size:0.94em;
+white-space:nowrap;
+border:1px inset Gray;
+padding:3px;
+}
+table.summary td {
+text-align:right;
+white-space:nowrap;
+border:1px inset Gray;
+padding:3px 5px 3px 10px;
+}
+span.mngl {
+padding-left:1em;
+font-size:0.875em;
+cursor:text;
+color:#44;
+font-weight:bold;
+}
+span.pleft {
+padding-left:2.5em;
+}
+span.color_p {
+font-style:italic;
+color:Brown;
+}
+span.param {
+font-style:italic;
+}
+span.focus_p {
+font-style:italic;
+background-color:#DCDCDC;
+}
+span.ttype {
+font-weight:normal;
+}
+span.nowrap {
+white-space:nowrap;
+}
+span.value {
+white-space:nowrap;
+font-weight:bold;
+}
+.passed {
+background-color:#CCFFCC;
+font-weight:normal;
+}
+.warning {
+background-color:#F4F4AF;
+font-weight:normal;
+}
+.failed {
+background-color:#FF;
+font-weight:normal;
+}
+.new {
+background-color:#C6DEFF;
+font-weight:normal;
+}
+
+.compatible {
+background-color:#CCFFCC;
+font-weight:normal;
+}
+.almost_compatible {
+background-color:#FFDAA3;
+font-weight:normal;
+}
+.incompatible {
+background-color:#FF;
+font-weight:normal;
+}
+.gray {
+background-color:#DCDCDC;
+font-weight:normal;
+}
+
+.top_ref {
+font-size:0.69em;
+}
+.footer {
+font-size:0.8125em;
+}
+.tabset {
+float:left;
+}
+a.tab {
+border:1px solid Black;
+float:left;
+margin:0px 5px -1px 0px;
+padding:3px 5px 3px 5px;
+position:relative;
+font-size:0.875em;
+background-color:#DDD;
+text-decoration:none;
+color:Black;
+}
+a.disabled:hover
+{
+color:Black;
+background:#EEE;
+}
+a.active:hover
+{
+color:Black;
+background:White;
+}
+a.active {
+border-bottom-color:White;
+background-color:White;
+}
+div.tab {
+
[hbase] branch branch-2.4 updated (999cc74bd59 -> 0bcbc02b5b4)
This is an automated email from the ASF dual-hosted git repository. huaxiangsun pushed a change to branch branch-2.4 in repository https://gitbox.apache.org/repos/asf/hbase.git from 999cc74bd59 HBASE-27089 Add “commons.crypto.stream.buffer.size” configuration (#4491) add f507fb71c67 Preparing hbase release 2.4.14RC0; tagging and updates to CHANGES.md and RELEASENOTES.md new 0bcbc02b5b4 Preparing development version 2.4.15-SNAPSHOT The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: CHANGES.md | 90 ++ RELEASENOTES.md| 90 ++ hbase-annotations/pom.xml | 2 +- hbase-archetypes/hbase-archetype-builder/pom.xml | 2 +- hbase-archetypes/hbase-client-project/pom.xml | 2 +- .../hbase-shaded-client-project/pom.xml| 2 +- hbase-archetypes/pom.xml | 2 +- hbase-assembly/pom.xml | 2 +- hbase-asyncfs/pom.xml | 2 +- hbase-build-configuration/pom.xml | 2 +- hbase-checkstyle/pom.xml | 4 +- hbase-client/pom.xml | 2 +- hbase-common/pom.xml | 2 +- hbase-endpoint/pom.xml | 2 +- hbase-examples/pom.xml | 2 +- hbase-external-blockcache/pom.xml | 2 +- hbase-hadoop-compat/pom.xml| 2 +- hbase-hadoop2-compat/pom.xml | 2 +- hbase-hbtop/pom.xml| 2 +- hbase-http/pom.xml | 2 +- hbase-it/pom.xml | 2 +- hbase-logging/pom.xml | 2 +- hbase-mapreduce/pom.xml| 2 +- hbase-metrics-api/pom.xml | 2 +- hbase-metrics/pom.xml | 2 +- hbase-procedure/pom.xml| 2 +- hbase-protocol-shaded/pom.xml | 2 +- hbase-protocol/pom.xml | 2 +- hbase-replication/pom.xml | 2 +- hbase-resource-bundle/pom.xml | 2 +- hbase-rest/pom.xml | 2 +- hbase-rsgroup/pom.xml | 2 +- hbase-server/pom.xml | 2 +- hbase-shaded/hbase-shaded-check-invariants/pom.xml | 2 +- .../hbase-shaded-client-byo-hadoop/pom.xml | 2 +- hbase-shaded/hbase-shaded-client/pom.xml | 2 +- hbase-shaded/hbase-shaded-mapreduce/pom.xml| 2 +- .../hbase-shaded-testing-util-tester/pom.xml | 2 +- hbase-shaded/hbase-shaded-testing-util/pom.xml | 2 +- .../pom.xml| 2 +- hbase-shaded/pom.xml | 2 +- hbase-shell/pom.xml| 2 +- hbase-testing-util/pom.xml | 2 +- hbase-thrift/pom.xml | 2 +- hbase-zookeeper/pom.xml| 2 +- pom.xml| 2 +- 46 files changed, 225 insertions(+), 45 deletions(-)
[hbase] 01/01: Preparing development version 2.4.15-SNAPSHOT
This is an automated email from the ASF dual-hosted git repository. huaxiangsun pushed a commit to branch branch-2.4 in repository https://gitbox.apache.org/repos/asf/hbase.git commit 0bcbc02b5b46ec94bcc109e09844a6c9bcfc0fb0 Author: Huaxiang Sun AuthorDate: Mon Aug 22 19:38:35 2022 + Preparing development version 2.4.15-SNAPSHOT Signed-off-by: Huaxiang Sun --- hbase-annotations/pom.xml | 2 +- hbase-archetypes/hbase-archetype-builder/pom.xml | 2 +- hbase-archetypes/hbase-client-project/pom.xml | 2 +- hbase-archetypes/hbase-shaded-client-project/pom.xml | 2 +- hbase-archetypes/pom.xml | 2 +- hbase-assembly/pom.xml | 2 +- hbase-asyncfs/pom.xml | 2 +- hbase-build-configuration/pom.xml | 2 +- hbase-checkstyle/pom.xml | 4 ++-- hbase-client/pom.xml | 2 +- hbase-common/pom.xml | 2 +- hbase-endpoint/pom.xml | 2 +- hbase-examples/pom.xml | 2 +- hbase-external-blockcache/pom.xml | 2 +- hbase-hadoop-compat/pom.xml| 2 +- hbase-hadoop2-compat/pom.xml | 2 +- hbase-hbtop/pom.xml| 2 +- hbase-http/pom.xml | 2 +- hbase-it/pom.xml | 2 +- hbase-logging/pom.xml | 2 +- hbase-mapreduce/pom.xml| 2 +- hbase-metrics-api/pom.xml | 2 +- hbase-metrics/pom.xml | 2 +- hbase-procedure/pom.xml| 2 +- hbase-protocol-shaded/pom.xml | 2 +- hbase-protocol/pom.xml | 2 +- hbase-replication/pom.xml | 2 +- hbase-resource-bundle/pom.xml | 2 +- hbase-rest/pom.xml | 2 +- hbase-rsgroup/pom.xml | 2 +- hbase-server/pom.xml | 2 +- hbase-shaded/hbase-shaded-check-invariants/pom.xml | 2 +- hbase-shaded/hbase-shaded-client-byo-hadoop/pom.xml| 2 +- hbase-shaded/hbase-shaded-client/pom.xml | 2 +- hbase-shaded/hbase-shaded-mapreduce/pom.xml| 2 +- hbase-shaded/hbase-shaded-testing-util-tester/pom.xml | 2 +- hbase-shaded/hbase-shaded-testing-util/pom.xml | 2 +- hbase-shaded/hbase-shaded-with-hadoop-check-invariants/pom.xml | 2 +- hbase-shaded/pom.xml | 2 +- hbase-shell/pom.xml| 2 +- hbase-testing-util/pom.xml | 2 +- hbase-thrift/pom.xml | 2 +- hbase-zookeeper/pom.xml| 2 +- pom.xml| 2 +- 44 files changed, 45 insertions(+), 45 deletions(-) diff --git a/hbase-annotations/pom.xml b/hbase-annotations/pom.xml index 37ca10415b9..bb67a145282 100644 --- a/hbase-annotations/pom.xml +++ b/hbase-annotations/pom.xml @@ -23,7 +23,7 @@ org.apache.hbase hbase -2.4.14 +2.4.15-SNAPSHOT .. diff --git a/hbase-archetypes/hbase-archetype-builder/pom.xml b/hbase-archetypes/hbase-archetype-builder/pom.xml index b67dbb29109..777d45c2787 100644 --- a/hbase-archetypes/hbase-archetype-builder/pom.xml +++ b/hbase-archetypes/hbase-archetype-builder/pom.xml @@ -24,7 +24,7 @@ org.apache.hbase hbase-archetypes -2.4.14 +2.4.15-SNAPSHOT .. diff --git a/hbase-archetypes/hbase-client-project/pom.xml b/hbase-archetypes/hbase-client-project/pom.xml index 9cec3a4934d..b15e57b384d 100644 --- a/hbase-archetypes/hbase-client-project/pom.xml +++ b/hbase-archetypes/hbase-client-project/pom.xml @@ -23,7 +23,7 @@ org.apache.hbase hbase-archetypes -2.4.14 +2.4.15-SNAPSHOT .. hbase-client-project diff --git a/hbase-archetypes/hbase-shaded-client-project/pom.xml b/hbase-archetypes/hbase-shaded-client-project/pom.xml index fb5e3f945ca..c13da88bffe 100644 --- a/hbase-archetypes/hbase-shaded-client-project/pom.xml +++ b/hbase-archetypes/hbase-shaded-client-project/pom.xml @@ -23,7 +23,7 @@ org.apache.hbase hbase-archetypes -2.4.14 +2.4.15-SNAPSHOT ..
[hbase] 01/01: Preparing hbase release 2.4.14RC0; tagging and updates to CHANGES.md and RELEASENOTES.md
This is an automated email from the ASF dual-hosted git repository. huaxiangsun pushed a commit to annotated tag 2.4.14RC0 in repository https://gitbox.apache.org/repos/asf/hbase.git commit f507fb71c6724ed44eb4ac25bbdd3e35b2441a1a Author: Huaxiang Sun AuthorDate: Mon Aug 22 19:38:24 2022 + Preparing hbase release 2.4.14RC0; tagging and updates to CHANGES.md and RELEASENOTES.md Signed-off-by: Huaxiang Sun --- CHANGES.md | 90 ++ RELEASENOTES.md| 90 ++ hbase-annotations/pom.xml | 2 +- hbase-archetypes/hbase-archetype-builder/pom.xml | 2 +- hbase-archetypes/hbase-client-project/pom.xml | 2 +- .../hbase-shaded-client-project/pom.xml| 2 +- hbase-archetypes/pom.xml | 2 +- hbase-assembly/pom.xml | 2 +- hbase-asyncfs/pom.xml | 2 +- hbase-build-configuration/pom.xml | 2 +- hbase-checkstyle/pom.xml | 4 +- hbase-client/pom.xml | 2 +- hbase-common/pom.xml | 2 +- hbase-endpoint/pom.xml | 2 +- hbase-examples/pom.xml | 2 +- hbase-external-blockcache/pom.xml | 2 +- hbase-hadoop-compat/pom.xml| 2 +- hbase-hadoop2-compat/pom.xml | 2 +- hbase-hbtop/pom.xml| 2 +- hbase-http/pom.xml | 2 +- hbase-it/pom.xml | 2 +- hbase-logging/pom.xml | 2 +- hbase-mapreduce/pom.xml| 2 +- hbase-metrics-api/pom.xml | 2 +- hbase-metrics/pom.xml | 2 +- hbase-procedure/pom.xml| 2 +- hbase-protocol-shaded/pom.xml | 2 +- hbase-protocol/pom.xml | 2 +- hbase-replication/pom.xml | 2 +- hbase-resource-bundle/pom.xml | 2 +- hbase-rest/pom.xml | 2 +- hbase-rsgroup/pom.xml | 2 +- hbase-server/pom.xml | 2 +- hbase-shaded/hbase-shaded-check-invariants/pom.xml | 2 +- .../hbase-shaded-client-byo-hadoop/pom.xml | 2 +- hbase-shaded/hbase-shaded-client/pom.xml | 2 +- hbase-shaded/hbase-shaded-mapreduce/pom.xml| 2 +- .../hbase-shaded-testing-util-tester/pom.xml | 2 +- hbase-shaded/hbase-shaded-testing-util/pom.xml | 2 +- .../pom.xml| 2 +- hbase-shaded/pom.xml | 2 +- hbase-shell/pom.xml| 2 +- hbase-testing-util/pom.xml | 2 +- hbase-thrift/pom.xml | 2 +- hbase-zookeeper/pom.xml| 2 +- pom.xml| 2 +- 46 files changed, 225 insertions(+), 45 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2e77dc51d99..406d296d2d7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -20,6 +20,96 @@ # Be careful doing manual edits in this file. Do not change format # of release header or remove the below marker. This file is generated. # DO NOT REMOVE THIS MARKER; FOR INTERPOLATING CHANGES!--> +## Release 2.4.14 - Unreleased (as of 2022-08-22) + + + +### NEW FEATURES: + +| JIRA | Summary | Priority | Component | +|: |: | :--- |: | +| [HBASE-27129](https://issues.apache.org/jira/browse/HBASE-27129) | Add a config that allows us to configure region-level storage policies | Major | regionserver | + + +### IMPROVEMENTS: + +| JIRA | Summary | Priority | Component | +|: |: | :--- |: | +| [HBASE-27089](https://issues.apache.org/jira/browse/HBASE-27089) | Add “commons.crypto.stream.buffer.size” configuration | Minor | io | +| [HBASE-27268](https://issues.apache.org/jira/browse/HBASE-27268) | In trace log mode, the client does not print callId/startTime and the server does not print receiveTime | Minor | logging | +| [HBASE-27296](https://issues.apache.org/jira/browse/HBASE-27296) | Some Cell's implementation of toString() such as IndividualBytesFieldCell prints out value and tags which is too verbose | Minor | logging | +| [HBASE-27273](https://issues.apache.org/jira/browse/HBASE-27273) | Should stop autoRead and skip all the bytes when rpc request too big | Major | IPC/RPC | +| [HBASE-27257](https://issues.apache.org/jira/browse/HBASE-27257) | Remove unnecessary usage of CachedBlocksByFile from RS UI | Major | . | +| [HBASE-27225](https://issues.apache.org/jira/browse/HBASE-27225) | Add BucketAllocator bucket
[hbase] annotated tag 2.4.14RC0 created (now 417d0732b48)
This is an automated email from the ASF dual-hosted git repository. huaxiangsun pushed a change to annotated tag 2.4.14RC0 in repository https://gitbox.apache.org/repos/asf/hbase.git at 417d0732b48 (tag) tagging f507fb71c6724ed44eb4ac25bbdd3e35b2441a1a (commit) replaces rel/2.4.13 by Huaxiang Sun on Mon Aug 22 19:38:24 2022 + - Log - Via create-release -BEGIN PGP SIGNATURE- iQIzBAABCgAdFiEEZQ5ukLH6mbYnhA++I6kA1BF8g14FAmMD2zAACgkQI6kA1BF8 g16mpRAAqMSBelk+htHx6ZOv0YrXkOnI/g6J6Irsos90shNn8UdFcAZ2Qq06Rlxg hOf2hWwfN663A3Fl5g4td9rlt07H6AnZu2HzHWeNgADex/nAZ1A31wu7dCOVqsek dMCsQTN6qOn23r5XxwNKayqP3lNMAtGfy/hLLC2V+vydDEXN5z/s8scu1deO3SGx sUXRFPamrpe4y5Jizr/yxjwPiXjHCkJlz4Fs/AZT32uuIOrcMOHRAXmI/gsYLofA fBVsfY8tFOmzt4TWCKqNrsqhMj2aKRfQP919gQZ/U6hG4NsHR7cyI6iWpdllQ7sF VtVINJOApDr+y62+qoJMq6ehjCfM5jzz15SDGgI5B94osQLYObn6lZAiiEaxDsWk +2w3zRFUERBe0MfTsU3yXc90NB1hSpPkWx4XBZJLgSK39NTNgWkrr86Hk1j+JBMK N4wzS6a0sOqEZjt3RbgfjcCKw4nzKyGwjfGfVAXnS0Kag0HDluU/oijMXYY7fc/r fr6mO/Cb/kr1eq66Er7XqhjMlATpB4h79PwfLNT44V3TdlLQWJHs0OtSm2SFr0pR SjNYWCKgZ7nMA709mGloEB6wt7mlvijdol7ihGMJ1ElFC9wxaMA1WtrTo3S2NDDD Qp5O2xOLpHixSIwvccQbYkom0kkhj0Vmtmc/2YK3ASuBQvOr/f0= =CFv8 -END PGP SIGNATURE- --- This annotated tag includes the following new commits: new f507fb71c67 Preparing hbase release 2.4.14RC0; tagging and updates to CHANGES.md and RELEASENOTES.md The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[hbase] branch branch-2 updated: HBASE-27305 add an option to skip file splitting when bulkload hfiles (#4709)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new f66d67bbdfb HBASE-27305 add an option to skip file splitting when
bulkload hfiles (#4709)
f66d67bbdfb is described below
commit f66d67bbdfbe9be96a28023b8019c24d79c05274
Author: Ruanhui <32773751+frostr...@users.noreply.github.com>
AuthorDate: Mon Aug 22 21:14:30 2022 +0800
HBASE-27305 add an option to skip file splitting when bulkload hfiles
(#4709)
Co-authored-by: huiruan
Signed-off-by: Duo Zhang
(cherry picked from commit 00a719e76f16b7380f6695fc986f003e0e5f47fe)
Conflicts:
hbase-server/src/main/java/org/apache/hadoop/hbase/tool/BulkLoadHFilesTool.java
---
.../hadoop/hbase/tool/LoadIncrementalHFiles.java | 11 +
.../hbase/tool/TestLoadIncrementalHFiles.java | 26 ++
2 files changed, 37 insertions(+)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/LoadIncrementalHFiles.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/LoadIncrementalHFiles.java
index 7842e21d076..5e3f2e9468f 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/LoadIncrementalHFiles.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/LoadIncrementalHFiles.java
@@ -141,6 +141,9 @@ public class LoadIncrementalHFiles extends Configured
implements Tool {
BulkLoadHFiles.IGNORE_UNMATCHED_CF_CONF_KEY;
public final static String ALWAYS_COPY_FILES =
BulkLoadHFiles.ALWAYS_COPY_FILES;
+ public static final String FAIL_IF_NEED_SPLIT_HFILE =
+"hbase.loadincremental.fail.if.need.split.hfile";
+
// We use a '.' prefix which is ignored when walking directory trees
// above. It is invalid family name.
static final String TMP_DIR = ".tmp";
@@ -162,6 +165,8 @@ public class LoadIncrementalHFiles extends Configured
implements Tool {
private boolean replicate = true;
+ private boolean failIfNeedSplitHFile = false;
+
/**
* Represents an HFile waiting to be loaded. An queue is used in this class
in order to support
* the case where a region has split during the process of the load. When
this happens, the HFile
@@ -195,6 +200,7 @@ public class LoadIncrementalHFiles extends Configured
implements Tool {
assignSeqIds = conf.getBoolean(ASSIGN_SEQ_IDS, true);
maxFilesPerRegionPerFamily = conf.getInt(MAX_FILES_PER_REGION_PER_FAMILY,
32);
bulkLoadByFamily =
conf.getBoolean(BulkLoadHFiles.BULK_LOAD_HFILES_BY_FAMILY, false);
+failIfNeedSplitHFile = conf.getBoolean(FAIL_IF_NEED_SPLIT_HFILE, false);
nrThreads =
conf.getInt("hbase.loadincremental.threads.max",
Runtime.getRuntime().availableProcessors());
numRetries = new AtomicInteger(0);
@@ -803,6 +809,11 @@ public class LoadIncrementalHFiles extends Configured
implements Tool {
Bytes.compareTo(last.get(), startEndKeys.getSecond()[firstKeyRegionIdx])
< 0
|| Bytes.equals(startEndKeys.getSecond()[firstKeyRegionIdx],
HConstants.EMPTY_BYTE_ARRAY);
if (!lastKeyInRange) {
+ if (failIfNeedSplitHFile) {
+throw new IOException(
+ "The key range of hfile=" + hfilePath + " fits into no region. " +
"And because "
++ FAIL_IF_NEED_SPLIT_HFILE + " was set to true, we just skip the
next steps.");
+ }
int lastKeyRegionIdx = getRegionIndex(startEndKeys, last.get());
int splitIdx = (firstKeyRegionIdx + lastKeyRegionIdx) >>> 1;
// make sure the splitPoint is valid in case region overlap occur, maybe
the splitPoint bigger
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFiles.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFiles.java
index a842c1d223f..cef666942f8 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFiles.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFiles.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.tool;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -793,4 +794,29 @@ public class TestLoadIncrementalHFiles {
util.getConfiguration().setBoolean(BulkLoadHFiles.BULK_LOAD_HFILES_BY_FAMILY,
false);
}
}
+
+ @Test
+ public void testFailIfNeedSplitHFile() throws IOException {
+TableName tableName = TableName.valueOf(tn.getMethodName());
+Table table = util.createTable(tableName, FAMILY);
+
+util.loadTable(table, FAMILY);
+
+FileSystem fs = util.getTestFileSystem();
+Path sfPath = new Path(fs.getWorkingDirectory(), new
Path(Bytes.toString(FAMILY), "file"));
+
[hbase] branch branch-2.5 updated: HBASE-27305 add an option to skip file splitting when bulkload hfiles (#4709)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.5 by this push:
new 51576ae45a9 HBASE-27305 add an option to skip file splitting when
bulkload hfiles (#4709)
51576ae45a9 is described below
commit 51576ae45a97036e9b0a578d742d27c89ea0e808
Author: Ruanhui <32773751+frostr...@users.noreply.github.com>
AuthorDate: Mon Aug 22 21:14:30 2022 +0800
HBASE-27305 add an option to skip file splitting when bulkload hfiles
(#4709)
Co-authored-by: huiruan
Signed-off-by: Duo Zhang
(cherry picked from commit 00a719e76f16b7380f6695fc986f003e0e5f47fe)
Conflicts:
hbase-server/src/main/java/org/apache/hadoop/hbase/tool/BulkLoadHFilesTool.java
---
.../hadoop/hbase/tool/LoadIncrementalHFiles.java | 11 +
.../hbase/tool/TestLoadIncrementalHFiles.java | 26 ++
2 files changed, 37 insertions(+)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/LoadIncrementalHFiles.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/LoadIncrementalHFiles.java
index 7842e21d076..5e3f2e9468f 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/LoadIncrementalHFiles.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/LoadIncrementalHFiles.java
@@ -141,6 +141,9 @@ public class LoadIncrementalHFiles extends Configured
implements Tool {
BulkLoadHFiles.IGNORE_UNMATCHED_CF_CONF_KEY;
public final static String ALWAYS_COPY_FILES =
BulkLoadHFiles.ALWAYS_COPY_FILES;
+ public static final String FAIL_IF_NEED_SPLIT_HFILE =
+"hbase.loadincremental.fail.if.need.split.hfile";
+
// We use a '.' prefix which is ignored when walking directory trees
// above. It is invalid family name.
static final String TMP_DIR = ".tmp";
@@ -162,6 +165,8 @@ public class LoadIncrementalHFiles extends Configured
implements Tool {
private boolean replicate = true;
+ private boolean failIfNeedSplitHFile = false;
+
/**
* Represents an HFile waiting to be loaded. An queue is used in this class
in order to support
* the case where a region has split during the process of the load. When
this happens, the HFile
@@ -195,6 +200,7 @@ public class LoadIncrementalHFiles extends Configured
implements Tool {
assignSeqIds = conf.getBoolean(ASSIGN_SEQ_IDS, true);
maxFilesPerRegionPerFamily = conf.getInt(MAX_FILES_PER_REGION_PER_FAMILY,
32);
bulkLoadByFamily =
conf.getBoolean(BulkLoadHFiles.BULK_LOAD_HFILES_BY_FAMILY, false);
+failIfNeedSplitHFile = conf.getBoolean(FAIL_IF_NEED_SPLIT_HFILE, false);
nrThreads =
conf.getInt("hbase.loadincremental.threads.max",
Runtime.getRuntime().availableProcessors());
numRetries = new AtomicInteger(0);
@@ -803,6 +809,11 @@ public class LoadIncrementalHFiles extends Configured
implements Tool {
Bytes.compareTo(last.get(), startEndKeys.getSecond()[firstKeyRegionIdx])
< 0
|| Bytes.equals(startEndKeys.getSecond()[firstKeyRegionIdx],
HConstants.EMPTY_BYTE_ARRAY);
if (!lastKeyInRange) {
+ if (failIfNeedSplitHFile) {
+throw new IOException(
+ "The key range of hfile=" + hfilePath + " fits into no region. " +
"And because "
++ FAIL_IF_NEED_SPLIT_HFILE + " was set to true, we just skip the
next steps.");
+ }
int lastKeyRegionIdx = getRegionIndex(startEndKeys, last.get());
int splitIdx = (firstKeyRegionIdx + lastKeyRegionIdx) >>> 1;
// make sure the splitPoint is valid in case region overlap occur, maybe
the splitPoint bigger
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFiles.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFiles.java
index a842c1d223f..cef666942f8 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFiles.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFiles.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.tool;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -793,4 +794,29 @@ public class TestLoadIncrementalHFiles {
util.getConfiguration().setBoolean(BulkLoadHFiles.BULK_LOAD_HFILES_BY_FAMILY,
false);
}
}
+
+ @Test
+ public void testFailIfNeedSplitHFile() throws IOException {
+TableName tableName = TableName.valueOf(tn.getMethodName());
+Table table = util.createTable(tableName, FAMILY);
+
+util.loadTable(table, FAMILY);
+
+FileSystem fs = util.getTestFileSystem();
+Path sfPath = new Path(fs.getWorkingDirectory(), new
Path(Bytes.toString(FAMILY), "file"));
+
[hbase-site] branch asf-site updated: INFRA-10751 Empty commit
This is an automated email from the ASF dual-hosted git repository. git-site-role pushed a commit to branch asf-site in repository https://gitbox.apache.org/repos/asf/hbase-site.git The following commit(s) were added to refs/heads/asf-site by this push: new 8a5a398e53 INFRA-10751 Empty commit 8a5a398e53 is described below commit 8a5a398e53654db59b1fbffea9642d02b18fd9e2 Author: jenkins AuthorDate: Mon Aug 22 14:42:18 2022 + INFRA-10751 Empty commit
[hbase] branch branch-2.5 updated: HBASE-27104 Add a tool command list_unknownservers (#4714)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.5 by this push:
new 78f587f5596 HBASE-27104 Add a tool command list_unknownservers (#4714)
78f587f5596 is described below
commit 78f587f5596df5558c56e49c7654a5612fd67245
Author: LiangJun He <2005hit...@163.com>
AuthorDate: Mon Aug 22 21:15:47 2022 +0800
HBASE-27104 Add a tool command list_unknownservers (#4714)
Signed-off-by: Duo Zhang
(cherry picked from commit 1bd0b581cb68341d6087f9fda0e4f3efe289a3a0)
---
.../org/apache/hadoop/hbase/ClusterMetrics.java| 7 ++
.../apache/hadoop/hbase/ClusterMetricsBuilder.java | 46 --
.../org/apache/hadoop/hbase/ClusterStatus.java | 5 ++
.../java/org/apache/hadoop/hbase/client/Admin.java | 8 ++
.../org/apache/hadoop/hbase/client/AsyncAdmin.java | 8 ++
.../hadoop/hbase/client/AsyncHBaseAdmin.java | 5 ++
.../src/main/protobuf/ClusterStatus.proto | 2 +
.../org/apache/hadoop/hbase/master/HMaster.java| 17
.../hbase/master/TestRegionsRecoveryChore.java | 5 ++
.../hadoop/hbase/master/TestUnknownServers.java| 98 ++
hbase-shell/src/main/ruby/hbase/admin.rb | 8 +-
hbase-shell/src/main/ruby/shell.rb | 1 +
.../ruby/shell/commands/list_unknownservers.rb | 44 ++
13 files changed, 245 insertions(+), 9 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetrics.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetrics.java
index 8be97fa1fda..a8a1493c349 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetrics.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetrics.java
@@ -76,6 +76,9 @@ public interface ClusterMetrics {
/** Returns the names of region servers on the dead list */
List getDeadServerNames();
+ /** Returns the names of region servers on the unknown list */
+ List getUnknownServerNames();
+
/** Returns the names of region servers on the live list */
Map getLiveServerMetrics();
@@ -176,6 +179,10 @@ public interface ClusterMetrics {
* metrics about dead region servers
*/
DEAD_SERVERS,
+/**
+ * metrics about unknown region servers
+ */
+UNKNOWN_SERVERS,
/**
* metrics about master name
*/
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetricsBuilder.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetricsBuilder.java
index 630e3620a67..9ca65463e02 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetricsBuilder.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetricsBuilder.java
@@ -47,6 +47,8 @@ public final class ClusterMetricsBuilder {
.collect(Collectors.toList()))
.addAllDeadServers(metrics.getDeadServerNames().stream().map(ProtobufUtil::toServerName)
.collect(Collectors.toList()))
+.addAllUnknownServers(metrics.getUnknownServerNames().stream()
+ .map(ProtobufUtil::toServerName).collect(Collectors.toList()))
.addAllLiveServers(metrics.getLiveServerMetrics().entrySet().stream()
.map(s -> ClusterStatusProtos.LiveServerInfo.newBuilder()
.setServer(ProtobufUtil.toServerName(s.getKey()))
@@ -98,6 +100,8 @@ public final class ClusterMetricsBuilder {
ServerMetricsBuilder::toServerMetrics)))
.setDeadServerNames(proto.getDeadServersList().stream().map(ProtobufUtil::toServerName)
.collect(Collectors.toList()))
+
.setUnknownServerNames(proto.getUnknownServersList().stream().map(ProtobufUtil::toServerName)
+.collect(Collectors.toList()))
.setBackerMasterNames(proto.getBackupMastersList().stream().map(ProtobufUtil::toServerName)
.collect(Collectors.toList()))
.setRegionsInTransition(proto.getRegionsInTransitionList().stream()
@@ -147,6 +151,8 @@ public final class ClusterMetricsBuilder {
return ClusterMetrics.Option.LIVE_SERVERS;
case DEAD_SERVERS:
return ClusterMetrics.Option.DEAD_SERVERS;
+ case UNKNOWN_SERVERS:
+return ClusterMetrics.Option.UNKNOWN_SERVERS;
case REGIONS_IN_TRANSITION:
return ClusterMetrics.Option.REGIONS_IN_TRANSITION;
case CLUSTER_ID:
@@ -186,6 +192,8 @@ public final class ClusterMetricsBuilder {
return ClusterStatusProtos.Option.LIVE_SERVERS;
case DEAD_SERVERS:
return ClusterStatusProtos.Option.DEAD_SERVERS;
+ case UNKNOWN_SERVERS:
+return ClusterStatusProtos.Option.UNKNOWN_SERVERS;
case REGIONS_IN_TRANSITION:
return ClusterStatusProtos.Option.REGIONS_IN_TRANSITION;
case CLUSTER_ID:
@@ -238,6 +246,7 @@ public final class ClusterMetricsBuilder {
@Nullable
private String
[hbase] branch branch-2 updated: HBASE-27104 Add a tool command list_unknownservers (#4714)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 1bd0b581cb6 HBASE-27104 Add a tool command list_unknownservers (#4714)
1bd0b581cb6 is described below
commit 1bd0b581cb68341d6087f9fda0e4f3efe289a3a0
Author: LiangJun He <2005hit...@163.com>
AuthorDate: Mon Aug 22 21:15:47 2022 +0800
HBASE-27104 Add a tool command list_unknownservers (#4714)
Signed-off-by: Duo Zhang
---
.../org/apache/hadoop/hbase/ClusterMetrics.java| 7 ++
.../apache/hadoop/hbase/ClusterMetricsBuilder.java | 46 --
.../org/apache/hadoop/hbase/ClusterStatus.java | 5 ++
.../java/org/apache/hadoop/hbase/client/Admin.java | 8 ++
.../org/apache/hadoop/hbase/client/AsyncAdmin.java | 8 ++
.../hadoop/hbase/client/AsyncHBaseAdmin.java | 5 ++
.../src/main/protobuf/ClusterStatus.proto | 2 +
.../org/apache/hadoop/hbase/master/HMaster.java| 17
.../hbase/master/TestRegionsRecoveryChore.java | 5 ++
.../hadoop/hbase/master/TestUnknownServers.java| 98 ++
hbase-shell/src/main/ruby/hbase/admin.rb | 8 +-
hbase-shell/src/main/ruby/shell.rb | 1 +
.../ruby/shell/commands/list_unknownservers.rb | 44 ++
13 files changed, 245 insertions(+), 9 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetrics.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetrics.java
index 8be97fa1fda..a8a1493c349 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetrics.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetrics.java
@@ -76,6 +76,9 @@ public interface ClusterMetrics {
/** Returns the names of region servers on the dead list */
List getDeadServerNames();
+ /** Returns the names of region servers on the unknown list */
+ List getUnknownServerNames();
+
/** Returns the names of region servers on the live list */
Map getLiveServerMetrics();
@@ -176,6 +179,10 @@ public interface ClusterMetrics {
* metrics about dead region servers
*/
DEAD_SERVERS,
+/**
+ * metrics about unknown region servers
+ */
+UNKNOWN_SERVERS,
/**
* metrics about master name
*/
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetricsBuilder.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetricsBuilder.java
index 630e3620a67..9ca65463e02 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetricsBuilder.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetricsBuilder.java
@@ -47,6 +47,8 @@ public final class ClusterMetricsBuilder {
.collect(Collectors.toList()))
.addAllDeadServers(metrics.getDeadServerNames().stream().map(ProtobufUtil::toServerName)
.collect(Collectors.toList()))
+.addAllUnknownServers(metrics.getUnknownServerNames().stream()
+ .map(ProtobufUtil::toServerName).collect(Collectors.toList()))
.addAllLiveServers(metrics.getLiveServerMetrics().entrySet().stream()
.map(s -> ClusterStatusProtos.LiveServerInfo.newBuilder()
.setServer(ProtobufUtil.toServerName(s.getKey()))
@@ -98,6 +100,8 @@ public final class ClusterMetricsBuilder {
ServerMetricsBuilder::toServerMetrics)))
.setDeadServerNames(proto.getDeadServersList().stream().map(ProtobufUtil::toServerName)
.collect(Collectors.toList()))
+
.setUnknownServerNames(proto.getUnknownServersList().stream().map(ProtobufUtil::toServerName)
+.collect(Collectors.toList()))
.setBackerMasterNames(proto.getBackupMastersList().stream().map(ProtobufUtil::toServerName)
.collect(Collectors.toList()))
.setRegionsInTransition(proto.getRegionsInTransitionList().stream()
@@ -147,6 +151,8 @@ public final class ClusterMetricsBuilder {
return ClusterMetrics.Option.LIVE_SERVERS;
case DEAD_SERVERS:
return ClusterMetrics.Option.DEAD_SERVERS;
+ case UNKNOWN_SERVERS:
+return ClusterMetrics.Option.UNKNOWN_SERVERS;
case REGIONS_IN_TRANSITION:
return ClusterMetrics.Option.REGIONS_IN_TRANSITION;
case CLUSTER_ID:
@@ -186,6 +192,8 @@ public final class ClusterMetricsBuilder {
return ClusterStatusProtos.Option.LIVE_SERVERS;
case DEAD_SERVERS:
return ClusterStatusProtos.Option.DEAD_SERVERS;
+ case UNKNOWN_SERVERS:
+return ClusterStatusProtos.Option.UNKNOWN_SERVERS;
case REGIONS_IN_TRANSITION:
return ClusterStatusProtos.Option.REGIONS_IN_TRANSITION;
case CLUSTER_ID:
@@ -238,6 +246,7 @@ public final class ClusterMetricsBuilder {
@Nullable
private String hbaseVersion;
private List deadServerNames = Collections.emptyList();
+
[hbase] branch master updated: HBASE-27305 add an option to skip file splitting when bulkload hfiles (#4709)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 00a719e76f1 HBASE-27305 add an option to skip file splitting when
bulkload hfiles (#4709)
00a719e76f1 is described below
commit 00a719e76f16b7380f6695fc986f003e0e5f47fe
Author: Ruanhui <32773751+frostr...@users.noreply.github.com>
AuthorDate: Mon Aug 22 21:14:30 2022 +0800
HBASE-27305 add an option to skip file splitting when bulkload hfiles
(#4709)
Co-authored-by: huiruan
Signed-off-by: Duo Zhang
---
.../hadoop/hbase/tool/BulkLoadHFilesTool.java | 10 +
.../hadoop/hbase/tool/TestBulkLoadHFiles.java | 26 ++
2 files changed, 36 insertions(+)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/BulkLoadHFilesTool.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/BulkLoadHFilesTool.java
index d1c99fc6334..06f97cf0aff 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/BulkLoadHFilesTool.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/BulkLoadHFilesTool.java
@@ -124,6 +124,9 @@ public class BulkLoadHFilesTool extends Configured
implements BulkLoadHFiles, To
*/
public static final String BULK_LOAD_HFILES_BY_FAMILY =
"hbase.mapreduce.bulkload.by.family";
+ public static final String FAIL_IF_NEED_SPLIT_HFILE =
+"hbase.loadincremental.fail.if.need.split.hfile";
+
// We use a '.' prefix which is ignored when walking directory trees
// above. It is invalid family name.
static final String TMP_DIR = ".tmp";
@@ -141,6 +144,7 @@ public class BulkLoadHFilesTool extends Configured
implements BulkLoadHFiles, To
private List clusterIds = new ArrayList<>();
private boolean replicate = true;
+ private boolean failIfNeedSplitHFile = false;
public BulkLoadHFilesTool(Configuration conf) {
// make a copy, just to be sure we're not overriding someone else's config
@@ -159,6 +163,7 @@ public class BulkLoadHFilesTool extends Configured
implements BulkLoadHFiles, To
nrThreads =
conf.getInt("hbase.loadincremental.threads.max",
Runtime.getRuntime().availableProcessors());
bulkLoadByFamily = conf.getBoolean(BULK_LOAD_HFILES_BY_FAMILY, false);
+failIfNeedSplitHFile = conf.getBoolean(FAIL_IF_NEED_SPLIT_HFILE, false);
}
// Initialize a thread pool
@@ -699,6 +704,11 @@ public class BulkLoadHFilesTool extends Configured
implements BulkLoadHFiles, To
Bytes.compareTo(last.get(),
startEndKeys.get(firstKeyRegionIdx).getSecond()) < 0 || Bytes
.equals(startEndKeys.get(firstKeyRegionIdx).getSecond(),
HConstants.EMPTY_BYTE_ARRAY);
if (!lastKeyInRange) {
+ if (failIfNeedSplitHFile) {
+throw new IOException(
+ "The key range of hfile=" + hfilePath + " fits into no region. " +
"And because "
++ FAIL_IF_NEED_SPLIT_HFILE + " was set to true, we just skip the
next steps.");
+ }
int lastKeyRegionIdx = getRegionIndex(startEndKeys, last.get());
int splitIdx = (firstKeyRegionIdx + lastKeyRegionIdx) / 2;
// make sure the splitPoint is valid in case region overlap occur, maybe
the splitPoint bigger
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestBulkLoadHFiles.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestBulkLoadHFiles.java
index f15ba688b13..591d807c0da 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestBulkLoadHFiles.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestBulkLoadHFiles.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.tool;
import static org.apache.hadoop.hbase.HBaseTestingUtil.countRows;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -782,4 +783,29 @@ public class TestBulkLoadHFiles {
util.getConfiguration().setBoolean(BulkLoadHFilesTool.BULK_LOAD_HFILES_BY_FAMILY,
false);
}
}
+
+ @Test
+ public void testFailIfNeedSplitHFile() throws IOException {
+TableName tableName = TableName.valueOf(tn.getMethodName());
+Table table = util.createTable(tableName, FAMILY);
+
+util.loadTable(table, FAMILY);
+
+FileSystem fs = util.getTestFileSystem();
+Path sfPath = new Path(fs.getWorkingDirectory(), new
Path(Bytes.toString(FAMILY), "file"));
+HFileTestUtil.createHFile(util.getConfiguration(), fs, sfPath, FAMILY,
QUALIFIER,
+ Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), 1000);
+
+util.getAdmin().split(tableName);
+util.waitFor(1, 1000, () ->
util.getAdmin().getRegions(tableName).size() > 1);
+
+Configuration config = new Configuration(util.getConfiguration());
+
[hbase] branch HBASE-27109/table_based_rqs updated (6dd64f1218c -> ad9d11753cd)
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a change to branch HBASE-27109/table_based_rqs in repository https://gitbox.apache.org/repos/asf/hbase.git discard 6dd64f1218c HBASE-27213 Add support for claim queue operation (#4708) discard 7605530b4a4 HBASE-27212 Implement a new table based replication queue storage and make the minimum replication system work (#4672) add f238a92b6df HBASE-27129 Add a config that allows us to configure region-level storage policies (#4547) new 37f92c068cc HBASE-27212 Implement a new table based replication queue storage and make the minimum replication system work (#4672) new ad9d11753cd HBASE-27213 Add support for claim queue operation (#4708) This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (6dd64f1218c) \ N -- N -- N refs/heads/HBASE-27109/table_based_rqs (ad9d11753cd) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../java/org/apache/hadoop/hbase/regionserver/HRegion.java | 6 ++ .../hadoop/hbase/regionserver/HRegionFileSystem.java | 14 ++ 2 files changed, 20 insertions(+)
[hbase] 02/02: HBASE-27213 Add support for claim queue operation (#4708)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-27109/table_based_rqs
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit ad9d11753cda6213894e1c03aefb7215cfcdea9f
Author: Duo Zhang
AuthorDate: Sat Aug 20 23:10:58 2022 +0800
HBASE-27213 Add support for claim queue operation (#4708)
Signed-off-by: Xin Sun
---
.../protobuf/server/master/MasterProcedure.proto | 6 +--
.../AssignReplicationQueuesProcedure.java | 13 ++---
.../master/replication/ModifyPeerProcedure.java| 2 +-
.../master/replication/RemovePeerProcedure.java| 41 +-
.../regionserver/ReplicationSourceManager.java | 37 +
.../replication/TestClaimReplicationQueue.java | 2 +-
...java => TestRemovePeerProcedureWaitForSCP.java} | 63 +-
.../replication/TestSerialReplicationFailover.java | 3 --
8 files changed, 116 insertions(+), 51 deletions(-)
diff --git
a/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
b/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
index 46dd2d32d3c..fe572f3663c 100644
---
a/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
+++
b/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
@@ -515,6 +515,7 @@ message UpdatePeerConfigStateData {
message RemovePeerStateData {
optional ReplicationPeer peer_config = 1;
+ repeated int64 ongoing_assign_replication_queues_proc_ids = 2;
}
message EnablePeerStateData {
@@ -711,9 +712,8 @@ message ModifyColumnFamilyStoreFileTrackerStateData {
}
enum AssignReplicationQueuesState {
- ASSIGN_REPLICATION_QUEUES_PRE_CHECK = 1;
- ASSIGN_REPLICATION_QUEUES_ADD_MISSING_QUEUES = 2;
- ASSIGN_REPLICATION_QUEUES_CLAIM = 3;
+ ASSIGN_REPLICATION_QUEUES_ADD_MISSING_QUEUES = 1;
+ ASSIGN_REPLICATION_QUEUES_CLAIM = 2;
}
message AssignReplicationQueuesStateData {
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AssignReplicationQueuesProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AssignReplicationQueuesProcedure.java
index e7fb5e51715..d33259dd436 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AssignReplicationQueuesProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AssignReplicationQueuesProcedure.java
@@ -23,6 +23,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.procedure.ServerProcedureInterface;
@@ -102,8 +103,12 @@ public class AssignReplicationQueuesProcedure
}
private Flow claimQueues(MasterProcedureEnv env) throws ReplicationException
{
+Set existingPeerIds =
env.getReplicationPeerManager().listPeers(null).stream()
+ .map(ReplicationPeerDescription::getPeerId).collect(Collectors.toSet());
ReplicationQueueStorage storage =
env.getReplicationPeerManager().getQueueStorage();
-List queueIds = storage.listAllQueueIds(crashedServer);
+// filter out replication queue for deleted peers
+List queueIds =
storage.listAllQueueIds(crashedServer).stream()
+ .filter(q ->
existingPeerIds.contains(q.getPeerId())).collect(Collectors.toList());
if (queueIds.isEmpty()) {
LOG.debug("Finish claiming replication queues for {}", crashedServer);
// we are done
@@ -130,10 +135,6 @@ public class AssignReplicationQueuesProcedure
throws ProcedureSuspendedException, ProcedureYieldException,
InterruptedException {
try {
switch (state) {
-case ASSIGN_REPLICATION_QUEUES_PRE_CHECK:
- // TODO: reserved for implementing the fencing logic with
Add/Remove/UpdatePeerProcedure
-
setNextState(AssignReplicationQueuesState.ASSIGN_REPLICATION_QUEUES_ADD_MISSING_QUEUES);
- return Flow.HAS_MORE_STATE;
case ASSIGN_REPLICATION_QUEUES_ADD_MISSING_QUEUES:
addMissingQueues(env);
retryCounter = null;
@@ -183,7 +184,7 @@ public class AssignReplicationQueuesProcedure
@Override
protected AssignReplicationQueuesState getInitialState() {
-return AssignReplicationQueuesState.ASSIGN_REPLICATION_QUEUES_PRE_CHECK;
+return
AssignReplicationQueuesState.ASSIGN_REPLICATION_QUEUES_ADD_MISSING_QUEUES;
}
@Override
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
index ae64e8850bb..f9d1cb1903b 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
+++
