[hbase-operator-tools] branch master updated (90e0f45 -> 308838b)
This is an automated email from the ASF dual-hosted git repository. zyork pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hbase-operator-tools.git. from 90e0f45 HBASE-26072 Upgrade hbase version in hbase-operator-tools to 2.4.4 (#91) add 308838b HBASE-26257 Improve Performance of HBCK when specifying a subset of tables (#92) No new revisions were added by this update. Summary of changes: .../java/org/apache/hbase/hbck1/HBaseFsck.java | 60 ++ 1 file changed, 38 insertions(+), 22 deletions(-)
[hbase-site] branch asf-site updated: INFRA-10751 Empty commit
This is an automated email from the ASF dual-hosted git repository. git-site-role pushed a commit to branch asf-site in repository https://gitbox.apache.org/repos/asf/hbase-site.git The following commit(s) were added to refs/heads/asf-site by this push: new bc4cd55 INFRA-10751 Empty commit bc4cd55 is described below commit bc4cd55fcc9803c384d5a9d828c67970be4f7369 Author: jenkins AuthorDate: Thu Sep 9 20:17:36 2021 + INFRA-10751 Empty commit
[hbase] branch branch-2 updated: HBASE-26240 Set InterfaceAudience to Private for BalanceRequest$Builder (#3663)
This is an automated email from the ASF dual-hosted git repository.
ndimiduk pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new bb05901 HBASE-26240 Set InterfaceAudience to Private for
BalanceRequest$Builder (#3663)
bb05901 is described below
commit bb059019ace4d5029cce1f70185baa0d93e41efd
Author: Bryan Beaudreault
AuthorDate: Thu Sep 9 13:25:20 2021 -0400
HBASE-26240 Set InterfaceAudience to Private for BalanceRequest$Builder
(#3663)
Signed-off-by: Duo Zhang
Signed-off-by: Nick Dimiduk
---
.../main/java/org/apache/hadoop/hbase/client/BalanceRequest.java | 8
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BalanceRequest.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BalanceRequest.java
index e464410..03ce5db 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BalanceRequest.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BalanceRequest.java
@@ -30,7 +30,7 @@ public final class BalanceRequest {
/**
* Builder for constructing a {@link BalanceRequest}
*/
- @InterfaceAudience.Public
+ @InterfaceAudience.Private
public final static class Builder {
private boolean dryRun = false;
private boolean ignoreRegionsInTransition = false;
@@ -38,7 +38,7 @@ public final class BalanceRequest {
private Builder() {}
/**
- * Creates a BalancerRequest which runs the balancer in dryRun mode.
+ * Updates BalancerRequest to run the balancer in dryRun mode.
* In this mode, the balancer will try to find a plan but WILL NOT
* execute any region moves or call any coprocessors.
*
@@ -55,8 +55,8 @@ public final class BalanceRequest {
}
/**
- * Creates a BalancerRequest to cause the balancer to run even if there
- * are regions in transition.
+ * Updates BalancerRequest to run the balancer even if there are regions
+ * in transition.
*
* WARNING: Advanced usage only, this could cause more issues than it
fixes.
*/
[hbase] branch branch-2.4 updated: HBASE-26255 Add an option to use region location from meta table in TableSnapshotInputFormat (#3661) (#3669)
This is an automated email from the ASF dual-hosted git repository.
huaxiangsun pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new be23541 HBASE-26255 Add an option to use region location from meta
table in TableSnapshotInputFormat (#3661) (#3669)
be23541 is described below
commit be23541f1182c355d4ae367607abeb5d8a97d06d
Author: huaxiangsun
AuthorDate: Thu Sep 9 10:43:53 2021 -0700
HBASE-26255 Add an option to use region location from meta table in
TableSnapshotInputFormat (#3661) (#3669)
Signed-off-by: Anoop Sam John
---
.../mapreduce/TableSnapshotInputFormatImpl.java| 61 ++
.../mapreduce/TestTableSnapshotInputFormat.java| 29 ++
2 files changed, 79 insertions(+), 11 deletions(-)
diff --git
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatImpl.java
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatImpl.java
index 5ed9be6..c3f05f4 100644
---
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatImpl.java
+++
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatImpl.java
@@ -32,9 +32,13 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HDFSBlocksDistribution;
import org.apache.hadoop.hbase.HDFSBlocksDistribution.HostAndWeight;
import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.PrivateCellUtil;
import org.apache.hadoop.hbase.client.ClientSideRegionScanner;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.IsolationLevel;
+import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.TableDescriptor;
@@ -102,6 +106,15 @@ public class TableSnapshotInputFormatImpl {
public static final boolean SNAPSHOT_INPUTFORMAT_LOCALITY_ENABLED_DEFAULT =
true;
/**
+ * Whether to calculate the Snapshot region location by region location from
meta.
+ * It is much faster than computing block locations for splits.
+ */
+ public static final String SNAPSHOT_INPUTFORMAT_LOCALITY_BY_REGION_LOCATION
=
+"hbase.TableSnapshotInputFormat.locality.by.region.location";
+
+ public static final boolean
SNAPSHOT_INPUTFORMAT_LOCALITY_BY_REGION_LOCATION_DEFAULT = false;
+
+ /**
* In some scenario, scan limited rows on each InputSplit for sampling data
extraction
*/
public static final String SNAPSHOT_INPUTFORMAT_ROW_LIMIT_PER_INPUTSPLIT =
@@ -392,17 +405,49 @@ public class TableSnapshotInputFormatImpl {
SNAPSHOT_INPUTFORMAT_SCAN_METRICS_ENABLED_DEFAULT);
scan.setScanMetricsEnabled(scanMetricsEnabled);
+boolean useRegionLoc =
conf.getBoolean(SNAPSHOT_INPUTFORMAT_LOCALITY_BY_REGION_LOCATION,
+ SNAPSHOT_INPUTFORMAT_LOCALITY_BY_REGION_LOCATION_DEFAULT);
+
+Connection connection = null;
+RegionLocator regionLocator = null;
+if (localityEnabled && useRegionLoc) {
+ Configuration newConf = new Configuration(conf);
+ newConf.setInt("hbase.hconnection.threads.max", 1);
+ try {
+connection = ConnectionFactory.createConnection(newConf);
+regionLocator = connection.getRegionLocator(htd.getTableName());
+
+/* Get all locations for the table and cache it */
+regionLocator.getAllRegionLocations();
+ } finally {
+if (connection != null) {
+ connection.close();
+}
+ }
+}
+
List splits = new ArrayList<>();
for (HRegionInfo hri : regionManifests) {
// load region descriptor
+ List hosts = null;
+ if (localityEnabled) {
+if (regionLocator != null) {
+ /* Get Location from the local cache */
+ HRegionLocation
+location = regionLocator.getRegionLocation(hri.getStartKey(),
false);
+
+ hosts = new ArrayList<>(1);
+ hosts.add(location.getHostname());
+} else {
+ hosts = calculateLocationsForInputSplit(conf, htd, hri, tableDir);
+}
+ }
if (numSplits > 1) {
byte[][] sp = sa.split(hri.getStartKey(), hri.getEndKey(), numSplits,
true);
for (int i = 0; i < sp.length - 1; i++) {
if (PrivateCellUtil.overlappingKeys(scan.getStartRow(),
scan.getStopRow(), sp[i],
sp[i + 1])) {
-List hosts =
-calculateLocationsForInputSplit(conf, htd, hri, tableDir,
localityEnabled);
Scan boundedScan = new Scan(scan);
if (scan.getStartRow().length == 0) {
@@ -425,8 +470,7 @@ public class TableSnapshotInputFormatImpl {
} else {
if (PrivateCellUti
[hbase] branch branch-2 updated: HBASE-26255 Add an option to use region location from meta table in TableSnapshotInputFormat (#3661) (#3668)
This is an automated email from the ASF dual-hosted git repository.
huaxiangsun pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 21d9741 HBASE-26255 Add an option to use region location from meta
table in TableSnapshotInputFormat (#3661) (#3668)
21d9741 is described below
commit 21d9741adb60fea71fdf1996233430a75a324968
Author: huaxiangsun
AuthorDate: Thu Sep 9 10:43:36 2021 -0700
HBASE-26255 Add an option to use region location from meta table in
TableSnapshotInputFormat (#3661) (#3668)
Signed-off-by: Anoop Sam John
---
.../mapreduce/TableSnapshotInputFormatImpl.java| 61 ++
.../mapreduce/TestTableSnapshotInputFormat.java| 29 ++
2 files changed, 79 insertions(+), 11 deletions(-)
diff --git
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatImpl.java
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatImpl.java
index 5ed9be6..c3f05f4 100644
---
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatImpl.java
+++
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatImpl.java
@@ -32,9 +32,13 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HDFSBlocksDistribution;
import org.apache.hadoop.hbase.HDFSBlocksDistribution.HostAndWeight;
import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.PrivateCellUtil;
import org.apache.hadoop.hbase.client.ClientSideRegionScanner;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.IsolationLevel;
+import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.TableDescriptor;
@@ -102,6 +106,15 @@ public class TableSnapshotInputFormatImpl {
public static final boolean SNAPSHOT_INPUTFORMAT_LOCALITY_ENABLED_DEFAULT =
true;
/**
+ * Whether to calculate the Snapshot region location by region location from
meta.
+ * It is much faster than computing block locations for splits.
+ */
+ public static final String SNAPSHOT_INPUTFORMAT_LOCALITY_BY_REGION_LOCATION
=
+"hbase.TableSnapshotInputFormat.locality.by.region.location";
+
+ public static final boolean
SNAPSHOT_INPUTFORMAT_LOCALITY_BY_REGION_LOCATION_DEFAULT = false;
+
+ /**
* In some scenario, scan limited rows on each InputSplit for sampling data
extraction
*/
public static final String SNAPSHOT_INPUTFORMAT_ROW_LIMIT_PER_INPUTSPLIT =
@@ -392,17 +405,49 @@ public class TableSnapshotInputFormatImpl {
SNAPSHOT_INPUTFORMAT_SCAN_METRICS_ENABLED_DEFAULT);
scan.setScanMetricsEnabled(scanMetricsEnabled);
+boolean useRegionLoc =
conf.getBoolean(SNAPSHOT_INPUTFORMAT_LOCALITY_BY_REGION_LOCATION,
+ SNAPSHOT_INPUTFORMAT_LOCALITY_BY_REGION_LOCATION_DEFAULT);
+
+Connection connection = null;
+RegionLocator regionLocator = null;
+if (localityEnabled && useRegionLoc) {
+ Configuration newConf = new Configuration(conf);
+ newConf.setInt("hbase.hconnection.threads.max", 1);
+ try {
+connection = ConnectionFactory.createConnection(newConf);
+regionLocator = connection.getRegionLocator(htd.getTableName());
+
+/* Get all locations for the table and cache it */
+regionLocator.getAllRegionLocations();
+ } finally {
+if (connection != null) {
+ connection.close();
+}
+ }
+}
+
List splits = new ArrayList<>();
for (HRegionInfo hri : regionManifests) {
// load region descriptor
+ List hosts = null;
+ if (localityEnabled) {
+if (regionLocator != null) {
+ /* Get Location from the local cache */
+ HRegionLocation
+location = regionLocator.getRegionLocation(hri.getStartKey(),
false);
+
+ hosts = new ArrayList<>(1);
+ hosts.add(location.getHostname());
+} else {
+ hosts = calculateLocationsForInputSplit(conf, htd, hri, tableDir);
+}
+ }
if (numSplits > 1) {
byte[][] sp = sa.split(hri.getStartKey(), hri.getEndKey(), numSplits,
true);
for (int i = 0; i < sp.length - 1; i++) {
if (PrivateCellUtil.overlappingKeys(scan.getStartRow(),
scan.getStopRow(), sp[i],
sp[i + 1])) {
-List hosts =
-calculateLocationsForInputSplit(conf, htd, hri, tableDir,
localityEnabled);
Scan boundedScan = new Scan(scan);
if (scan.getStartRow().length == 0) {
@@ -425,8 +470,7 @@ public class TableSnapshotInputFormatImpl {
} else {
if (PrivateCellUtil.ov
[hbase] branch branch-2.3 updated: HBASE-26255 Add an option to use region location from meta table in TableSnapshotInputFormat (#3661) (#3670)
This is an automated email from the ASF dual-hosted git repository.
huaxiangsun pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.3 by this push:
new 66fed7c HBASE-26255 Add an option to use region location from meta
table in TableSnapshotInputFormat (#3661) (#3670)
66fed7c is described below
commit 66fed7c743c3e247a9eb1ee323e3f7c3a7f609c2
Author: huaxiangsun
AuthorDate: Thu Sep 9 10:42:49 2021 -0700
HBASE-26255 Add an option to use region location from meta table in
TableSnapshotInputFormat (#3661) (#3670)
Signed-off-by: Anoop Sam John
---
.../mapreduce/TableSnapshotInputFormatImpl.java| 61 ++
.../mapreduce/TestTableSnapshotInputFormat.java| 29 ++
2 files changed, 79 insertions(+), 11 deletions(-)
diff --git
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatImpl.java
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatImpl.java
index 5ed9be6..c3f05f4 100644
---
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatImpl.java
+++
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatImpl.java
@@ -32,9 +32,13 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HDFSBlocksDistribution;
import org.apache.hadoop.hbase.HDFSBlocksDistribution.HostAndWeight;
import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.PrivateCellUtil;
import org.apache.hadoop.hbase.client.ClientSideRegionScanner;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.IsolationLevel;
+import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.TableDescriptor;
@@ -102,6 +106,15 @@ public class TableSnapshotInputFormatImpl {
public static final boolean SNAPSHOT_INPUTFORMAT_LOCALITY_ENABLED_DEFAULT =
true;
/**
+ * Whether to calculate the Snapshot region location by region location from
meta.
+ * It is much faster than computing block locations for splits.
+ */
+ public static final String SNAPSHOT_INPUTFORMAT_LOCALITY_BY_REGION_LOCATION
=
+"hbase.TableSnapshotInputFormat.locality.by.region.location";
+
+ public static final boolean
SNAPSHOT_INPUTFORMAT_LOCALITY_BY_REGION_LOCATION_DEFAULT = false;
+
+ /**
* In some scenario, scan limited rows on each InputSplit for sampling data
extraction
*/
public static final String SNAPSHOT_INPUTFORMAT_ROW_LIMIT_PER_INPUTSPLIT =
@@ -392,17 +405,49 @@ public class TableSnapshotInputFormatImpl {
SNAPSHOT_INPUTFORMAT_SCAN_METRICS_ENABLED_DEFAULT);
scan.setScanMetricsEnabled(scanMetricsEnabled);
+boolean useRegionLoc =
conf.getBoolean(SNAPSHOT_INPUTFORMAT_LOCALITY_BY_REGION_LOCATION,
+ SNAPSHOT_INPUTFORMAT_LOCALITY_BY_REGION_LOCATION_DEFAULT);
+
+Connection connection = null;
+RegionLocator regionLocator = null;
+if (localityEnabled && useRegionLoc) {
+ Configuration newConf = new Configuration(conf);
+ newConf.setInt("hbase.hconnection.threads.max", 1);
+ try {
+connection = ConnectionFactory.createConnection(newConf);
+regionLocator = connection.getRegionLocator(htd.getTableName());
+
+/* Get all locations for the table and cache it */
+regionLocator.getAllRegionLocations();
+ } finally {
+if (connection != null) {
+ connection.close();
+}
+ }
+}
+
List splits = new ArrayList<>();
for (HRegionInfo hri : regionManifests) {
// load region descriptor
+ List hosts = null;
+ if (localityEnabled) {
+if (regionLocator != null) {
+ /* Get Location from the local cache */
+ HRegionLocation
+location = regionLocator.getRegionLocation(hri.getStartKey(),
false);
+
+ hosts = new ArrayList<>(1);
+ hosts.add(location.getHostname());
+} else {
+ hosts = calculateLocationsForInputSplit(conf, htd, hri, tableDir);
+}
+ }
if (numSplits > 1) {
byte[][] sp = sa.split(hri.getStartKey(), hri.getEndKey(), numSplits,
true);
for (int i = 0; i < sp.length - 1; i++) {
if (PrivateCellUtil.overlappingKeys(scan.getStartRow(),
scan.getStopRow(), sp[i],
sp[i + 1])) {
-List hosts =
-calculateLocationsForInputSplit(conf, htd, hri, tableDir,
localityEnabled);
Scan boundedScan = new Scan(scan);
if (scan.getStartRow().length == 0) {
@@ -425,8 +470,7 @@ public class TableSnapshotInputFormatImpl {
} else {
if (PrivateCellUti
[hbase] branch master updated (8d2b995 -> 8679e08)
This is an automated email from the ASF dual-hosted git repository. ndimiduk pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hbase.git. from 8d2b995 HBASE-26229 Limit count and size of L0 files compaction in StripeCompactionPolicy (#3646) add 8679e08 HBASE-26240 Set InterfaceAudience to Private for BalanceRequest$Builder (#3663) No new revisions were added by this update. Summary of changes: .../main/java/org/apache/hadoop/hbase/client/BalanceRequest.java | 8 1 file changed, 4 insertions(+), 4 deletions(-)
[hbase] branch branch-2.4 updated: HBASE-26229 Limit count and size of L0 files compaction in StripeCompactionPolicy (#3646)
This is an automated email from the ASF dual-hosted git repository.
haxiaolin pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 466d2d5 HBASE-26229 Limit count and size of L0 files compaction in
StripeCompactionPolicy (#3646)
466d2d5 is described below
commit 466d2d55638f27d97dc25a9a96e21fab61e1db79
Author: Xiaolin Ha
AuthorDate: Thu Sep 9 18:26:37 2021 +0800
HBASE-26229 Limit count and size of L0 files compaction in
StripeCompactionPolicy (#3646)
Signed-off-by: Duo Zhang
---
.../compactions/ExploringCompactionPolicy.java | 22 +++-
.../compactions/StripeCompactionPolicy.java| 66 --
.../compactions/TestStripeCompactionPolicy.java| 40 -
3 files changed, 106 insertions(+), 22 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
index ce42530..76bf1d7 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
@@ -22,7 +22,6 @@ package org.apache.hadoop.hbase.regionserver.compactions;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.regionserver.HStoreFile;
import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
@@ -120,10 +119,29 @@ public class ExploringCompactionPolicy extends
RatioBasedCompactionPolicy {
}
LOG.debug("Exploring compaction algorithm has selected {} files of size
{} starting at " +
"candidate #{} after considering {} permutations with {} in ratio",
bestSelection.size(),
- bestSize, bestSize, opts, optsInRatio);
+ bestSize, bestStart, opts, optsInRatio);
return new ArrayList<>(bestSelection);
}
+ /**
+ * Select at least one file in the candidates list to compact, through
choosing files
+ * from the head to the index that the accumulation length larger the max
compaction size.
+ * This method is a supplementary of the selectSimpleCompaction() method,
aims to make sure
+ * at least one file can be selected to compact, for compactions like L0
files, which need to
+ * compact all files and as soon as possible.
+ */
+ public List selectCompactFiles(final List
candidates, int maxFiles,
+ boolean isOffpeak) {
+long selectedSize = 0L;
+for (int end = 0; end < Math.min(candidates.size(), maxFiles); end++) {
+ selectedSize += candidates.get(end).getReader().length();
+ if (selectedSize >= comConf.getMaxCompactSize(isOffpeak)) {
+return candidates.subList(0, end + 1);
+ }
+}
+return candidates;
+ }
+
private boolean isBetterSelection(List bestSelection, long
bestSize,
List selection, long size, boolean mightBeStuck) {
if (mightBeStuck && bestSize > 0 && size > 0) {
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/StripeCompactionPolicy.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/StripeCompactionPolicy.java
index 6804d36..3709521 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/StripeCompactionPolicy.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/StripeCompactionPolicy.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.hbase.regionserver.compactions;
import static
org.apache.hadoop.hbase.regionserver.StripeStoreFileManager.OPEN_KEY;
-
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
@@ -130,10 +129,12 @@ public class StripeCompactionPolicy extends
CompactionPolicy {
List l0Files = si.getLevel0Files();
// See if we need to make new stripes.
-boolean shouldCompactL0 = (this.config.getLevel0MinFiles() <=
l0Files.size());
+boolean shouldCompactL0 = this.config.getLevel0MinFiles() <=
l0Files.size();
if (stripeCount == 0) {
- if (!shouldCompactL0) return null; // nothing to do.
- return selectNewStripesCompaction(si);
+ if (!shouldCompactL0) {
+return null; // nothing to do.
+ }
+ return selectL0OnlyCompaction(si);
}
boolean canDropDeletesNoL0 = l0Files.isEmpty();
@@ -141,16 +142,20 @@ public class StripeCompactionPolicy extends
CompactionPolicy {
if (!canDropDeletesNoL0) {
// If we need to compact L0, see if we can add something to it, and
drop deletes.
StripeCompactionRequest result = selectSingleStripeCompaction(
-si, true, canDropDeletesNoL0, isOffpeak);
-if (result != null) retu
[hbase] branch branch-2 updated: HBASE-26229 Limit count and size of L0 files compaction in StripeCompactionPolicy (#3646)
This is an automated email from the ASF dual-hosted git repository.
haxiaolin pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new f670fc5 HBASE-26229 Limit count and size of L0 files compaction in
StripeCompactionPolicy (#3646)
f670fc5 is described below
commit f670fc5d4d1502f642fd749216fc1477eb8b5930
Author: Xiaolin Ha
AuthorDate: Thu Sep 9 18:26:37 2021 +0800
HBASE-26229 Limit count and size of L0 files compaction in
StripeCompactionPolicy (#3646)
Signed-off-by: Duo Zhang
---
.../compactions/ExploringCompactionPolicy.java | 22 +++-
.../compactions/StripeCompactionPolicy.java| 66 --
.../compactions/TestStripeCompactionPolicy.java| 40 -
3 files changed, 106 insertions(+), 22 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
index ce42530..76bf1d7 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
@@ -22,7 +22,6 @@ package org.apache.hadoop.hbase.regionserver.compactions;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.regionserver.HStoreFile;
import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
@@ -120,10 +119,29 @@ public class ExploringCompactionPolicy extends
RatioBasedCompactionPolicy {
}
LOG.debug("Exploring compaction algorithm has selected {} files of size
{} starting at " +
"candidate #{} after considering {} permutations with {} in ratio",
bestSelection.size(),
- bestSize, bestSize, opts, optsInRatio);
+ bestSize, bestStart, opts, optsInRatio);
return new ArrayList<>(bestSelection);
}
+ /**
+ * Select at least one file in the candidates list to compact, through
choosing files
+ * from the head to the index that the accumulation length larger the max
compaction size.
+ * This method is a supplementary of the selectSimpleCompaction() method,
aims to make sure
+ * at least one file can be selected to compact, for compactions like L0
files, which need to
+ * compact all files and as soon as possible.
+ */
+ public List selectCompactFiles(final List
candidates, int maxFiles,
+ boolean isOffpeak) {
+long selectedSize = 0L;
+for (int end = 0; end < Math.min(candidates.size(), maxFiles); end++) {
+ selectedSize += candidates.get(end).getReader().length();
+ if (selectedSize >= comConf.getMaxCompactSize(isOffpeak)) {
+return candidates.subList(0, end + 1);
+ }
+}
+return candidates;
+ }
+
private boolean isBetterSelection(List bestSelection, long
bestSize,
List selection, long size, boolean mightBeStuck) {
if (mightBeStuck && bestSize > 0 && size > 0) {
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/StripeCompactionPolicy.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/StripeCompactionPolicy.java
index 6804d36..3709521 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/StripeCompactionPolicy.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/StripeCompactionPolicy.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.hbase.regionserver.compactions;
import static
org.apache.hadoop.hbase.regionserver.StripeStoreFileManager.OPEN_KEY;
-
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
@@ -130,10 +129,12 @@ public class StripeCompactionPolicy extends
CompactionPolicy {
List l0Files = si.getLevel0Files();
// See if we need to make new stripes.
-boolean shouldCompactL0 = (this.config.getLevel0MinFiles() <=
l0Files.size());
+boolean shouldCompactL0 = this.config.getLevel0MinFiles() <=
l0Files.size();
if (stripeCount == 0) {
- if (!shouldCompactL0) return null; // nothing to do.
- return selectNewStripesCompaction(si);
+ if (!shouldCompactL0) {
+return null; // nothing to do.
+ }
+ return selectL0OnlyCompaction(si);
}
boolean canDropDeletesNoL0 = l0Files.isEmpty();
@@ -141,16 +142,20 @@ public class StripeCompactionPolicy extends
CompactionPolicy {
if (!canDropDeletesNoL0) {
// If we need to compact L0, see if we can add something to it, and
drop deletes.
StripeCompactionRequest result = selectSingleStripeCompaction(
-si, true, canDropDeletesNoL0, isOffpeak);
-if (result != null) return r
[hbase] branch master updated: HBASE-26229 Limit count and size of L0 files compaction in StripeCompactionPolicy (#3646)
This is an automated email from the ASF dual-hosted git repository.
haxiaolin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 8d2b995 HBASE-26229 Limit count and size of L0 files compaction in
StripeCompactionPolicy (#3646)
8d2b995 is described below
commit 8d2b995352d605f15ae359e65893033a9b1b3923
Author: Xiaolin Ha
AuthorDate: Thu Sep 9 18:26:37 2021 +0800
HBASE-26229 Limit count and size of L0 files compaction in
StripeCompactionPolicy (#3646)
Signed-off-by: Duo Zhang
---
.../compactions/ExploringCompactionPolicy.java | 22 +++-
.../compactions/StripeCompactionPolicy.java| 66 --
.../compactions/TestStripeCompactionPolicy.java| 40 -
3 files changed, 106 insertions(+), 22 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
index ce42530..76bf1d7 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
@@ -22,7 +22,6 @@ package org.apache.hadoop.hbase.regionserver.compactions;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.regionserver.HStoreFile;
import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
@@ -120,10 +119,29 @@ public class ExploringCompactionPolicy extends
RatioBasedCompactionPolicy {
}
LOG.debug("Exploring compaction algorithm has selected {} files of size
{} starting at " +
"candidate #{} after considering {} permutations with {} in ratio",
bestSelection.size(),
- bestSize, bestSize, opts, optsInRatio);
+ bestSize, bestStart, opts, optsInRatio);
return new ArrayList<>(bestSelection);
}
+ /**
+ * Select at least one file in the candidates list to compact, through
choosing files
+ * from the head to the index that the accumulation length larger the max
compaction size.
+ * This method is a supplementary of the selectSimpleCompaction() method,
aims to make sure
+ * at least one file can be selected to compact, for compactions like L0
files, which need to
+ * compact all files and as soon as possible.
+ */
+ public List selectCompactFiles(final List
candidates, int maxFiles,
+ boolean isOffpeak) {
+long selectedSize = 0L;
+for (int end = 0; end < Math.min(candidates.size(), maxFiles); end++) {
+ selectedSize += candidates.get(end).getReader().length();
+ if (selectedSize >= comConf.getMaxCompactSize(isOffpeak)) {
+return candidates.subList(0, end + 1);
+ }
+}
+return candidates;
+ }
+
private boolean isBetterSelection(List bestSelection, long
bestSize,
List selection, long size, boolean mightBeStuck) {
if (mightBeStuck && bestSize > 0 && size > 0) {
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/StripeCompactionPolicy.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/StripeCompactionPolicy.java
index 6804d36..3709521 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/StripeCompactionPolicy.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/StripeCompactionPolicy.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.hbase.regionserver.compactions;
import static
org.apache.hadoop.hbase.regionserver.StripeStoreFileManager.OPEN_KEY;
-
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
@@ -130,10 +129,12 @@ public class StripeCompactionPolicy extends
CompactionPolicy {
List l0Files = si.getLevel0Files();
// See if we need to make new stripes.
-boolean shouldCompactL0 = (this.config.getLevel0MinFiles() <=
l0Files.size());
+boolean shouldCompactL0 = this.config.getLevel0MinFiles() <=
l0Files.size();
if (stripeCount == 0) {
- if (!shouldCompactL0) return null; // nothing to do.
- return selectNewStripesCompaction(si);
+ if (!shouldCompactL0) {
+return null; // nothing to do.
+ }
+ return selectL0OnlyCompaction(si);
}
boolean canDropDeletesNoL0 = l0Files.isEmpty();
@@ -141,16 +142,20 @@ public class StripeCompactionPolicy extends
CompactionPolicy {
if (!canDropDeletesNoL0) {
// If we need to compact L0, see if we can add something to it, and
drop deletes.
StripeCompactionRequest result = selectSingleStripeCompaction(
-si, true, canDropDeletesNoL0, isOffpeak);
-if (result != null) return resul
