[hbase] 04/04: HBASE-26224 Introduce a MigrationStoreFileTracker to support migrating from different store file tracker implementations (#3656)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-26067
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 27fd631b52fff9f76b75b0e4bacaa5f0f9aafde7
Author: Duo Zhang
AuthorDate: Thu Sep 9 12:11:25 2021 +0800
HBASE-26224 Introduce a MigrationStoreFileTracker to support migrating from
different store file tracker implementations (#3656)
Signed-off-by: Wellington Chevreuil
---
.../storefiletracker/DefaultStoreFileTracker.java | 10 +-
.../FileBasedStoreFileTracker.java | 15 +-
.../MigrationStoreFileTracker.java | 88 ++
.../storefiletracker/StoreFileListFile.java| 6 +-
.../storefiletracker/StoreFileTrackerBase.java | 12 +-
.../storefiletracker/StoreFileTrackerFactory.java | 39 +++--
.../TestMigrationStoreFileTracker.java | 193 +
7 files changed, 343 insertions(+), 20 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java
index 22e0513..a13b75b 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.regionserver.storefiletracker;
import java.io.IOException;
import java.util.Collection;
+import java.util.Collections;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
@@ -39,7 +40,9 @@ class DefaultStoreFileTracker extends StoreFileTrackerBase {
@Override
public List load() throws IOException {
-return
ctx.getRegionFileSystem().getStoreFiles(ctx.getFamily().getNameAsString());
+List files =
+
ctx.getRegionFileSystem().getStoreFiles(ctx.getFamily().getNameAsString());
+return files != null ? files : Collections.emptyList();
}
@Override
@@ -57,4 +60,9 @@ class DefaultStoreFileTracker extends StoreFileTrackerBase {
Collection newFiles) throws IOException {
// NOOP
}
+
+ @Override
+ void set(List files) {
+// NOOP
+ }
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java
index de28b0e..c370b87 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java
@@ -48,7 +48,7 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.StoreFileTrackerProtos.
* storages.
*/
@InterfaceAudience.Private
-public class FileBasedStoreFileTracker extends StoreFileTrackerBase {
+class FileBasedStoreFileTracker extends StoreFileTrackerBase {
private final StoreFileListFile backedFile;
@@ -139,4 +139,17 @@ public class FileBasedStoreFileTracker extends
StoreFileTrackerBase {
}
}
}
+
+ @Override
+ void set(List files) throws IOException {
+synchronized (storefiles) {
+ storefiles.clear();
+ StoreFileList.Builder builder = StoreFileList.newBuilder();
+ for (StoreFileInfo info : files) {
+storefiles.put(info.getPath().getName(), info);
+builder.addStoreFile(toStoreFileEntry(info));
+ }
+ backedFile.update(builder);
+}
+ }
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java
new file mode 100644
index 000..e486e6d
--- /dev/null
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.region
[hbase] 03/04: HBASE-26079 Use StoreFileTracker when splitting and merging (#3617)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-26067
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 6731430f1a0c2ecd1388a1fa8dc844e4c377d74a
Author: Wellington Ramos Chevreuil
AuthorDate: Wed Sep 8 10:31:49 2021 +0100
HBASE-26079 Use StoreFileTracker when splitting and merging (#3617)
Signed-off-by: Duo Zhang
---
.../assignment/MergeTableRegionsProcedure.java | 23 +-
.../assignment/SplitTableRegionProcedure.java | 41 ++--
.../hbase/regionserver/HRegionFileSystem.java | 42 +++-
.../storefiletracker/DefaultStoreFileTracker.java | 4 +-
.../storefiletracker/StoreFileTracker.java | 1 -
.../storefiletracker/StoreFileTrackerFactory.java | 33 ++-
.../hbase/regionserver/TestDefaultStoreEngine.java | 1 +
.../regionserver/TestDirectStoreSplitsMerges.java | 32 ++-
.../hadoop/hbase/regionserver/TestHStoreFile.java | 19 +-
.../regionserver/TestMergesSplitsAddToTracker.java | 262 +
.../hbase/regionserver/TestStripeStoreEngine.java | 1 +
.../storefiletracker/TestStoreFileTracker.java}| 42 ++--
12 files changed, 436 insertions(+), 65 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
index fb57cb9..4b25a28 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
@@ -24,6 +24,8 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.stream.Stream;
+
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.MetaMutationAnnotation;
@@ -53,6 +55,8 @@ import org.apache.hadoop.hbase.quotas.QuotaExceededException;
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.HStoreFile;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
+import
org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.hbase.wal.WALSplitUtil;
@@ -584,40 +588,47 @@ public class MergeTableRegionsProcedure
final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
final Path tableDir = CommonFSUtils.getTableDir(mfs.getRootDir(),
regionsToMerge[0].getTable());
final FileSystem fs = mfs.getFileSystem();
-
+List mergedFiles = new ArrayList<>();
HRegionFileSystem mergeRegionFs =
HRegionFileSystem.createRegionOnFileSystem(
env.getMasterConfiguration(), fs, tableDir, mergedRegion);
for (RegionInfo ri: this.regionsToMerge) {
HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(
env.getMasterConfiguration(), fs, tableDir, ri, false);
- mergeStoreFiles(env, regionFs, mergeRegionFs, mergedRegion);
+ mergedFiles.addAll(mergeStoreFiles(env, regionFs, mergeRegionFs,
mergedRegion));
}
assert mergeRegionFs != null;
-mergeRegionFs.commitMergedRegion();
+mergeRegionFs.commitMergedRegion(mergedFiles, env);
// Prepare to create merged regions
env.getAssignmentManager().getRegionStates().
getOrCreateRegionStateNode(mergedRegion).setState(State.MERGING_NEW);
}
- private void mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem
regionFs,
+ private List mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem
regionFs,
HRegionFileSystem mergeRegionFs, RegionInfo mergedRegion) throws
IOException {
final TableDescriptor htd = env.getMasterServices().getTableDescriptors()
.get(mergedRegion.getTable());
+List mergedFiles = new ArrayList<>();
for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {
String family = hcd.getNameAsString();
- final Collection storeFiles =
regionFs.getStoreFiles(family);
+ Configuration trackerConfig =
+
StoreFileTrackerFactory.mergeConfigurations(env.getMasterConfiguration(), htd,
hcd);
+ StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig,
true,
+family, regionFs);
+ final Collection storeFiles = tracker.load();
if (storeFiles != null && storeFiles.size() > 0) {
for (StoreFileInfo storeFileInfo : storeFiles) {
// Create reference file(s) to parent region file here in mergedDir.
// As this procedure is running on master, use CacheConfig.DISABLED
means
// don't cache any block.
- mergeRegionFs.mergeStoreFi
[hbase] branch HBASE-26067 updated (b56fa39 -> 27fd631)
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a change to branch HBASE-26067 in repository https://gitbox.apache.org/repos/asf/hbase.git. discard b56fa39 HBASE-26224 Introduce a MigrationStoreFileTracker to support migrating from different store file tracker implementations (#3656) omit 0c8db60 HBASE-26079 Use StoreFileTracker when splitting and merging (#3617) omit f642ba3 HBASE-25988 Store the store file list by a file (#3578) omit df87f53 HBASE-26064 Introduce a StoreFileTracker to abstract the store file tracking logic add b8ba3f7 HBASE-26255 Add an option to use region location from meta table in TableSnapshotInputFormat (#3661) add 922d066 [HBASE-26243]Fix typo for file "hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java" (#3650) new 5492282 HBASE-26064 Introduce a StoreFileTracker to abstract the store file tracking logic new f3623e6 HBASE-25988 Store the store file list by a file (#3578) new 6731430 HBASE-26079 Use StoreFileTracker when splitting and merging (#3617) new 27fd631 HBASE-26224 Introduce a MigrationStoreFileTracker to support migrating from different store file tracker implementations (#3656) This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (b56fa39) \ N -- N -- N refs/heads/HBASE-26067 (27fd631) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. The 4 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../mapreduce/TableSnapshotInputFormatImpl.java| 63 +- .../mapreduce/TestTableSnapshotInputFormat.java| 29 ++ .../org/apache/hadoop/hbase/util/HBaseFsck.java| 6 +-- 3 files changed, 83 insertions(+), 15 deletions(-)
[hbase] 02/04: HBASE-25988 Store the store file list by a file (#3578)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-26067
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit f3623e604641469db2c33a425e15d5d08db532d4
Author: Duo Zhang
AuthorDate: Thu Aug 26 18:51:12 2021 +0800
HBASE-25988 Store the store file list by a file (#3578)
Signed-off-by: Wellington Chevreuil
---
.../protobuf/server/region/StoreFileTracker.proto | 29 +++--
.../hadoop/hbase/regionserver/StoreContext.java| 5 +
.../hadoop/hbase/regionserver/StoreEngine.java | 8 +-
.../storefiletracker/DefaultStoreFileTracker.java | 5 +-
.../FileBasedStoreFileTracker.java | 142 +
.../storefiletracker/StoreFileListFile.java| 142 +
.../storefiletracker/StoreFileTrackerBase.java | 11 +-
.../storefiletracker/StoreFileTrackerFactory.java | 12 +-
.../TestRegionWithFileBasedStoreFileTracker.java | 109
9 files changed, 430 insertions(+), 33 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java
b/hbase-protocol-shaded/src/main/protobuf/server/region/StoreFileTracker.proto
similarity index 57%
copy from
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java
copy to
hbase-protocol-shaded/src/main/protobuf/server/region/StoreFileTracker.proto
index 4f7231b..2a269ea 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java
+++
b/hbase-protocol-shaded/src/main/protobuf/server/region/StoreFileTracker.proto
@@ -15,21 +15,22 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.hbase.regionserver.storefiletracker;
+syntax = "proto2";
+// This file contains protocol buffers that are used for store file tracker.
+package hbase.pb;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.regionserver.StoreContext;
-import org.apache.yetus.audience.InterfaceAudience;
+option java_package = "org.apache.hadoop.hbase.shaded.protobuf.generated";
+option java_outer_classname = "StoreFileTrackerProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
-/**
- * Factory method for creating store file tracker.
- */
-@InterfaceAudience.Private
-public final class StoreFileTrackerFactory {
+message StoreFileEntry {
+ required string name = 1;
+ required uint64 size = 2;
+}
- public static StoreFileTracker create(Configuration conf, TableName
tableName,
-boolean isPrimaryReplica, StoreContext ctx) {
-return new DefaultStoreFileTracker(conf, tableName, isPrimaryReplica, ctx);
- }
+message StoreFileList {
+ required uint64 timestamp = 1;
+ repeated StoreFileEntry store_file = 2;
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java
index 2a9f968..588f8f4 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java
@@ -22,6 +22,7 @@ import java.util.Collection;
import java.util.function.Supplier;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.CellComparator;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.io.HeapSize;
@@ -109,6 +110,10 @@ public final class StoreContext implements HeapSize {
return coprocessorHost;
}
+ public TableName getTableName() {
+return getRegionInfo().getTable();
+ }
+
public RegionInfo getRegionInfo() {
return regionFileSystem.getRegionInfo();
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java
index f9d6c29..61b7dfa 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java
@@ -172,9 +172,9 @@ public abstract class StoreEnginehttp://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver.storefiletracker;
+
+import java.io.IOException;
+import ja
[hbase] 01/04: HBASE-26064 Introduce a StoreFileTracker to abstract the store file tracking logic
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-26067
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 54922827d58ced259f62675cde73045426053887
Author: Duo Zhang
AuthorDate: Thu Jul 29 18:35:19 2021 +0800
HBASE-26064 Introduce a StoreFileTracker to abstract the store file
tracking logic
Signed-off-by: Wellington Chevreuil
---
.../hadoop/hbase/mob/DefaultMobStoreCompactor.java | 22 +-
.../hadoop/hbase/mob/DefaultMobStoreFlusher.java | 4 +-
.../regionserver/CreateStoreFileWriterParams.java | 134
.../hbase/regionserver/DateTieredStoreEngine.java | 5 +-
.../hbase/regionserver/DefaultStoreEngine.java | 5 +-
.../hbase/regionserver/DefaultStoreFlusher.java| 11 +-
.../hadoop/hbase/regionserver/HMobStore.java | 3 +-
.../hbase/regionserver/HRegionFileSystem.java | 10 +-
.../apache/hadoop/hbase/regionserver/HStore.java | 699 +
.../hadoop/hbase/regionserver/StoreContext.java| 9 +
.../hadoop/hbase/regionserver/StoreEngine.java | 438 -
.../hbase/regionserver/StoreFileManager.java | 9 +
.../hadoop/hbase/regionserver/StoreFlusher.java| 9 +-
.../hadoop/hbase/regionserver/StoreUtils.java | 36 +-
.../hbase/regionserver/StripeStoreEngine.java | 9 +-
.../hbase/regionserver/StripeStoreFlusher.java | 9 +-
.../compactions/AbstractMultiOutputCompactor.java | 7 +-
.../hbase/regionserver/compactions/Compactor.java | 36 +-
.../regionserver/compactions/DefaultCompactor.java | 16 +-
.../storefiletracker/DefaultStoreFileTracker.java | 61 ++
.../storefiletracker/StoreFileTracker.java | 75 +++
.../storefiletracker/StoreFileTrackerBase.java | 178 ++
.../storefiletracker/StoreFileTrackerFactory.java | 35 ++
.../util/compaction/MajorCompactionRequest.java| 1 -
.../org/apache/hadoop/hbase/TestIOFencing.java | 12 +-
.../regionserver/TestCacheOnWriteInSchema.java | 6 +-
.../hbase/regionserver/TestDefaultStoreEngine.java | 5 +-
.../hadoop/hbase/regionserver/TestHRegion.java | 4 +-
.../hadoop/hbase/regionserver/TestHStore.java | 28 +-
.../TestRegionMergeTransactionOnCluster.java | 6 +-
.../regionserver/TestStoreFileRefresherChore.java | 3 +-
.../regionserver/TestStoreScannerClosure.java | 6 +-
.../hbase/regionserver/TestStripeStoreEngine.java | 2 +
.../compactions/TestDateTieredCompactor.java | 12 +-
.../compactions/TestStripeCompactionPolicy.java| 12 +-
.../compactions/TestStripeCompactor.java | 12 +-
36 files changed, 1224 insertions(+), 705 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
index c45fdff..01fe000 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
@@ -29,7 +29,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map.Entry;
import java.util.Optional;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -144,17 +143,16 @@ public class DefaultMobStoreCompactor extends
DefaultCompactor {
};
private final CellSinkFactory writerFactory =
- new CellSinkFactory() {
-@Override
-public StoreFileWriter createWriter(InternalScanner scanner,
-
org.apache.hadoop.hbase.regionserver.compactions.Compactor.FileDetails fd,
-boolean shouldDropBehind, boolean major) throws IOException {
- // make this writer with tags always because of possible new cells
with tags.
- return store.createWriterInTmp(fd.maxKeyCount,
-major ? majorCompactionCompression : minorCompactionCompression,
-true, true, true, shouldDropBehind);
-}
- };
+new CellSinkFactory() {
+ @Override
+ public StoreFileWriter createWriter(InternalScanner scanner,
+org.apache.hadoop.hbase.regionserver.compactions.Compactor.FileDetails
fd,
+boolean shouldDropBehind, boolean major) throws IOException {
+// make this writer with tags always because of possible new cells
with tags.
+return store.getStoreEngine().createWriter(
+ createParams(fd, shouldDropBehind,
major).includeMVCCReadpoint(true).includesTag(true));
+ }
+};
public DefaultMobStoreCompactor(Configuration conf, HStore store) {
super(conf, store);
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java
index 480b85c..4a1dc7b 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/D
[hbase] branch branch-2 updated: [HBASE-26243]Fix typo for file "hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java" (#3650)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 68db7f5 [HBASE-26243]Fix typo for file
"hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java" (#3650)
68db7f5 is described below
commit 68db7f5ed05c67d09d04035b242b959c48175623
Author: hapihu
AuthorDate: Thu Sep 9 12:11:49 2021 +0800
[HBASE-26243]Fix typo for file
"hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java" (#3650)
Signed-off-by: Duo Zhang
---
.../src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 4298831..8460998 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -1824,7 +1824,7 @@ public class HBaseFsck extends Configured implements
Closeable {
}
/**
- * Check consistency of all regions using mulitple threads concurrently.
+ * Check consistency of all regions using multiple threads concurrently.
*/
private void checkRegionConsistencyConcurrently(
final List workItems)
@@ -3564,7 +3564,7 @@ public class HBaseFsck extends Configured implements
Closeable {
out.println("NOTE: Following options are NOT supported as of HBase version
2.0+.");
out.println("");
out.println(" UNSUPPORTED Metadata Repair options: (expert features, use
with caution!)");
-out.println(" -fix Try to fix region assignments. This is
for backwards compatiblity");
+out.println(" -fix Try to fix region assignments. This is
for backwards compatibility");
out.println(" -fixAssignments Try to fix region assignments. Replaces
the old -fix");
out.println(" -fixMeta Try to fix meta problems. This assumes
HDFS region info is good.");
out.println(" -fixHdfsHoles Try to fix region holes in hdfs.");
@@ -3865,7 +3865,7 @@ public class HBaseFsck extends Configured implements
Closeable {
for (String arg : args) {
if (unsupportedOptionsInV2.contains(arg)) {
errors.reportError(ERROR_CODE.UNSUPPORTED_OPTION,
- "option '" + arg + "' is not " + "supportted!");
+ "option '" + arg + "' is not " + "supported!");
result = false;
break;
}
[hbase] branch branch-2.4 updated: [HBASE-26243]Fix typo for file "hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java" (#3650)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 9e04033 [HBASE-26243]Fix typo for file
"hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java" (#3650)
9e04033 is described below
commit 9e04033ad21cbc6e240431d47780119e579faec7
Author: hapihu
AuthorDate: Thu Sep 9 12:11:49 2021 +0800
[HBASE-26243]Fix typo for file
"hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java" (#3650)
Signed-off-by: Duo Zhang
---
.../src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index ba54d47..a0ee84b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -1823,7 +1823,7 @@ public class HBaseFsck extends Configured implements
Closeable {
}
/**
- * Check consistency of all regions using mulitple threads concurrently.
+ * Check consistency of all regions using multiple threads concurrently.
*/
private void checkRegionConsistencyConcurrently(
final List workItems)
@@ -3563,7 +3563,7 @@ public class HBaseFsck extends Configured implements
Closeable {
out.println("NOTE: Following options are NOT supported as of HBase version
2.0+.");
out.println("");
out.println(" UNSUPPORTED Metadata Repair options: (expert features, use
with caution!)");
-out.println(" -fix Try to fix region assignments. This is
for backwards compatiblity");
+out.println(" -fix Try to fix region assignments. This is
for backwards compatibility");
out.println(" -fixAssignments Try to fix region assignments. Replaces
the old -fix");
out.println(" -fixMeta Try to fix meta problems. This assumes
HDFS region info is good.");
out.println(" -fixHdfsHoles Try to fix region holes in hdfs.");
@@ -3864,7 +3864,7 @@ public class HBaseFsck extends Configured implements
Closeable {
for (String arg : args) {
if (unsupportedOptionsInV2.contains(arg)) {
errors.reportError(ERROR_CODE.UNSUPPORTED_OPTION,
- "option '" + arg + "' is not " + "supportted!");
+ "option '" + arg + "' is not " + "supported!");
result = false;
break;
}
[hbase] branch branch-2.3 updated: [HBASE-26243]Fix typo for file "hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java" (#3650)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.3 by this push:
new 250906e [HBASE-26243]Fix typo for file
"hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java" (#3650)
250906e is described below
commit 250906e82066b79c9023f05d512d3ce9076efa87
Author: hapihu
AuthorDate: Thu Sep 9 12:11:49 2021 +0800
[HBASE-26243]Fix typo for file
"hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java" (#3650)
Signed-off-by: Duo Zhang
---
.../src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index e964a23..150dcf9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -1822,7 +1822,7 @@ public class HBaseFsck extends Configured implements
Closeable {
}
/**
- * Check consistency of all regions using mulitple threads concurrently.
+ * Check consistency of all regions using multiple threads concurrently.
*/
private void checkRegionConsistencyConcurrently(
final List workItems)
@@ -3562,7 +3562,7 @@ public class HBaseFsck extends Configured implements
Closeable {
out.println("NOTE: Following options are NOT supported as of HBase version
2.0+.");
out.println("");
out.println(" UNSUPPORTED Metadata Repair options: (expert features, use
with caution!)");
-out.println(" -fix Try to fix region assignments. This is
for backwards compatiblity");
+out.println(" -fix Try to fix region assignments. This is
for backwards compatibility");
out.println(" -fixAssignments Try to fix region assignments. Replaces
the old -fix");
out.println(" -fixMeta Try to fix meta problems. This assumes
HDFS region info is good.");
out.println(" -fixHdfsHoles Try to fix region holes in hdfs.");
@@ -3863,7 +3863,7 @@ public class HBaseFsck extends Configured implements
Closeable {
for (String arg : args) {
if (unsupportedOptionsInV2.contains(arg)) {
errors.reportError(ERROR_CODE.UNSUPPORTED_OPTION,
- "option '" + arg + "' is not " + "supportted!");
+ "option '" + arg + "' is not " + "supported!");
result = false;
break;
}
[hbase] branch master updated: [HBASE-26243]Fix typo for file "hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java" (#3650)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 922d066 [HBASE-26243]Fix typo for file
"hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java" (#3650)
922d066 is described below
commit 922d0666a3d05f2e91fc38cb64750497564fc19d
Author: hapihu
AuthorDate: Thu Sep 9 12:11:49 2021 +0800
[HBASE-26243]Fix typo for file
"hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java" (#3650)
Signed-off-by: Duo Zhang
---
.../src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 28f0d5e..b91d7d93 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -1820,7 +1820,7 @@ public class HBaseFsck extends Configured implements
Closeable {
}
/**
- * Check consistency of all regions using mulitple threads concurrently.
+ * Check consistency of all regions using multiple threads concurrently.
*/
private void checkRegionConsistencyConcurrently(
final List workItems)
@@ -3559,7 +3559,7 @@ public class HBaseFsck extends Configured implements
Closeable {
out.println("NOTE: Following options are NOT supported as of HBase version
2.0+.");
out.println("");
out.println(" UNSUPPORTED Metadata Repair options: (expert features, use
with caution!)");
-out.println(" -fix Try to fix region assignments. This is
for backwards compatiblity");
+out.println(" -fix Try to fix region assignments. This is
for backwards compatibility");
out.println(" -fixAssignments Try to fix region assignments. Replaces
the old -fix");
out.println(" -fixMeta Try to fix meta problems. This assumes
HDFS region info is good.");
out.println(" -fixHdfsHoles Try to fix region holes in hdfs.");
@@ -3859,7 +3859,7 @@ public class HBaseFsck extends Configured implements
Closeable {
for (String arg : args) {
if (unsupportedOptionsInV2.contains(arg)) {
errors.reportError(ERROR_CODE.UNSUPPORTED_OPTION,
- "option '" + arg + "' is not " + "supportted!");
+ "option '" + arg + "' is not " + "supported!");
result = false;
break;
}
[hbase] branch HBASE-26067 updated: HBASE-26224 Introduce a MigrationStoreFileTracker to support migrating from different store file tracker implementations (#3656)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-26067
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/HBASE-26067 by this push:
new b56fa39 HBASE-26224 Introduce a MigrationStoreFileTracker to support
migrating from different store file tracker implementations (#3656)
b56fa39 is described below
commit b56fa39a85012416ed9edfe28ef49af4ab6f3d65
Author: Duo Zhang
AuthorDate: Thu Sep 9 12:11:25 2021 +0800
HBASE-26224 Introduce a MigrationStoreFileTracker to support migrating from
different store file tracker implementations (#3656)
Signed-off-by: Wellington Chevreuil
---
.../storefiletracker/DefaultStoreFileTracker.java | 10 +-
.../FileBasedStoreFileTracker.java | 15 +-
.../MigrationStoreFileTracker.java | 88 ++
.../storefiletracker/StoreFileListFile.java| 6 +-
.../storefiletracker/StoreFileTrackerBase.java | 12 +-
.../storefiletracker/StoreFileTrackerFactory.java | 39 +++--
.../TestMigrationStoreFileTracker.java | 193 +
7 files changed, 343 insertions(+), 20 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java
index 22e0513..a13b75b 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.regionserver.storefiletracker;
import java.io.IOException;
import java.util.Collection;
+import java.util.Collections;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
@@ -39,7 +40,9 @@ class DefaultStoreFileTracker extends StoreFileTrackerBase {
@Override
public List load() throws IOException {
-return
ctx.getRegionFileSystem().getStoreFiles(ctx.getFamily().getNameAsString());
+List files =
+
ctx.getRegionFileSystem().getStoreFiles(ctx.getFamily().getNameAsString());
+return files != null ? files : Collections.emptyList();
}
@Override
@@ -57,4 +60,9 @@ class DefaultStoreFileTracker extends StoreFileTrackerBase {
Collection newFiles) throws IOException {
// NOOP
}
+
+ @Override
+ void set(List files) {
+// NOOP
+ }
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java
index de28b0e..c370b87 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java
@@ -48,7 +48,7 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.StoreFileTrackerProtos.
* storages.
*/
@InterfaceAudience.Private
-public class FileBasedStoreFileTracker extends StoreFileTrackerBase {
+class FileBasedStoreFileTracker extends StoreFileTrackerBase {
private final StoreFileListFile backedFile;
@@ -139,4 +139,17 @@ public class FileBasedStoreFileTracker extends
StoreFileTrackerBase {
}
}
}
+
+ @Override
+ void set(List files) throws IOException {
+synchronized (storefiles) {
+ storefiles.clear();
+ StoreFileList.Builder builder = StoreFileList.newBuilder();
+ for (StoreFileInfo info : files) {
+storefiles.put(info.getPath().getName(), info);
+builder.addStoreFile(toStoreFileEntry(info));
+ }
+ backedFile.update(builder);
+}
+ }
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java
new file mode 100644
index 000..e486e6d
--- /dev/null
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distribut
[hbase-site] branch asf-site updated: INFRA-10751 Empty commit
This is an automated email from the ASF dual-hosted git repository. git-site-role pushed a commit to branch asf-site in repository https://gitbox.apache.org/repos/asf/hbase-site.git The following commit(s) were added to refs/heads/asf-site by this push: new 733111a INFRA-10751 Empty commit 733111a is described below commit 733111abcff39901bc85389727416a93656f0f5b Author: jenkins AuthorDate: Wed Sep 8 20:17:32 2021 + INFRA-10751 Empty commit
[hbase] branch branch-1 updated: HBASE-26195 Abort RS if wal sync fails or times out
This is an automated email from the ASF dual-hosted git repository.
gjacoby pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new d309276 HBASE-26195 Abort RS if wal sync fails or times out
d309276 is described below
commit d309276121fe00aa30c9b26bd0ac09214f4133d6
Author: Rushabh Shah
AuthorDate: Wed Sep 8 15:15:56 2021 -0400
HBASE-26195 Abort RS if wal sync fails or times out
---
.../apache/hadoop/hbase/regionserver/HRegion.java | 95 +++---
.../hbase/client/TestRollbackFromClient.java | 361 -
.../regionserver/TestFailedAppendAndSync.java | 1 +
.../hadoop/hbase/regionserver/TestHRegion.java | 60 +---
.../regionserver/wal/TestFSHLogTimedOutSync.java | 184 +++
5 files changed, 234 insertions(+), 467 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index b74731f..3e752cd 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3339,7 +3339,7 @@ public class HRegion implements HeapSize,
PropagatingConfigurationObserver, Regi
WALEdit walEdit = null;
MultiVersionConcurrencyControl.WriteEntry writeEntry = null;
long txid = 0;
-boolean doRollBackMemstore = false;
+boolean walSyncSuccess = true;
boolean locked = false;
int cellCount = 0;
/** Keep track of the locks we hold so we can release them in finally
clause */
@@ -3705,7 +3705,6 @@ public class HRegion implements HeapSize,
PropagatingConfigurationObserver, Regi
if (updateSeqId) {
updateSequenceId(familyMaps[i].values(), mvccNum);
}
-doRollBackMemstore = true; // If we have a failure, we need to clean
what we wrote
addedSize += applyFamilyMapToMemstore(familyMaps[i]);
}
@@ -3721,11 +3720,12 @@ public class HRegion implements HeapSize,
PropagatingConfigurationObserver, Regi
// -
// STEP 7. Sync wal.
// -
+ walSyncSuccess = false;
if (txid != 0) {
syncOrDefer(txid, durability);
}
+ walSyncSuccess = true;
- doRollBackMemstore = false;
// update memstore size
this.addAndGetGlobalMemstoreSize(addedSize);
@@ -3776,14 +3776,17 @@ public class HRegion implements HeapSize,
PropagatingConfigurationObserver, Regi
success = true;
return addedSize;
+} catch (Throwable t) {
+ // WAL sync failed. Aborting to avoid a mismatch between the memstore,
WAL,
+ // and any replicated clusters.
+ if (!walSyncSuccess) {
+rsServices.abort("WAL sync failed, aborting to preserve WAL as source
of truth", t);
+ }
+ // Rethrow the exception.
+ throw t;
} finally {
- // if the wal sync was unsuccessful, remove keys from memstore
- if (doRollBackMemstore) {
-for (int j = 0; j < familyMaps.length; j++) {
- for(List cells:familyMaps[j].values()) {
-rollbackMemstore(cells);
- }
-}
+ // if the wal sync was unsuccessful, complete the mvcc
+ if (!walSyncSuccess) {
if (writeEntry != null) mvcc.complete(writeEntry);
} else {
if (writeEntry != null) {
@@ -4255,33 +4258,9 @@ public class HRegion implements HeapSize,
PropagatingConfigurationObserver, Regi
Store store = getStore(family);
size += store.add(cells);
}
-
return size;
}
- /**
- * Remove all the keys listed in the map from the memstore. This method is
- * called when a Put/Delete has updated memstore but subsequently fails to
update
- * the wal. This method is then invoked to rollback the memstore.
- */
- private void rollbackMemstore(List memstoreCells) {
-rollbackMemstore(null, memstoreCells);
- }
-
- private void rollbackMemstore(final Store defaultStore, List
memstoreCells) {
-int kvsRolledback = 0;
-for (Cell cell : memstoreCells) {
- Store store = defaultStore;
- if (store == null) {
-byte[] family = CellUtil.cloneFamily(cell);
-store = getStore(family);
- }
- store.rollback(cell);
- kvsRolledback++;
-}
-LOG.debug("rollbackMemstore rolled back " + kvsRolledback);
- }
-
@Override
public void checkFamilies(Collection families) throws
NoSuchColumnFamilyException {
for (byte[] family : families) {
@@ -7964,7 +7943,7 @@ public class HRegion implements HeapSize,
PropagatingConfigurationObserver, Regi
this.writeRequestsCount.increment();
RowLock rowLock = null;
WALKey walKey = null;
-boolean doRollBackMemstore = false;
+boolean walSyncSuccess = true;
try {
rowLock = getRowLockInter
[hbase] branch master updated: HBASE-26255 Add an option to use region location from meta table in TableSnapshotInputFormat (#3661)
This is an automated email from the ASF dual-hosted git repository.
huaxiangsun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new b8ba3f7 HBASE-26255 Add an option to use region location from meta
table in TableSnapshotInputFormat (#3661)
b8ba3f7 is described below
commit b8ba3f788bac747d8266b90b6966cfdb8435450f
Author: huaxiangsun
AuthorDate: Wed Sep 8 09:46:34 2021 -0700
HBASE-26255 Add an option to use region location from meta table in
TableSnapshotInputFormat (#3661)
Signed-off-by: Anoop Sam John
---
.../mapreduce/TableSnapshotInputFormatImpl.java| 63 +-
.../mapreduce/TestTableSnapshotInputFormat.java| 29 ++
2 files changed, 80 insertions(+), 12 deletions(-)
diff --git
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatImpl.java
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatImpl.java
index abdd8f4..22c19be 100644
---
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatImpl.java
+++
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatImpl.java
@@ -31,10 +31,14 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HDFSBlocksDistribution;
import org.apache.hadoop.hbase.HDFSBlocksDistribution.HostAndWeight;
+import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.PrivateCellUtil;
import org.apache.hadoop.hbase.client.ClientSideRegionScanner;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.IsolationLevel;
import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.TableDescriptor;
@@ -102,6 +106,15 @@ public class TableSnapshotInputFormatImpl {
public static final boolean SNAPSHOT_INPUTFORMAT_LOCALITY_ENABLED_DEFAULT =
true;
/**
+ * Whether to calculate the Snapshot region location by region location from
meta.
+ * It is much faster than computing block locations for splits.
+ */
+ public static final String SNAPSHOT_INPUTFORMAT_LOCALITY_BY_REGION_LOCATION
=
+"hbase.TableSnapshotInputFormat.locality.by.region.location";
+
+ public static final boolean
SNAPSHOT_INPUTFORMAT_LOCALITY_BY_REGION_LOCATION_DEFAULT = false;
+
+ /**
* In some scenario, scan limited rows on each InputSplit for sampling data
extraction
*/
public static final String SNAPSHOT_INPUTFORMAT_ROW_LIMIT_PER_INPUTSPLIT =
@@ -392,17 +405,49 @@ public class TableSnapshotInputFormatImpl {
SNAPSHOT_INPUTFORMAT_SCAN_METRICS_ENABLED_DEFAULT);
scan.setScanMetricsEnabled(scanMetricsEnabled);
+boolean useRegionLoc =
conf.getBoolean(SNAPSHOT_INPUTFORMAT_LOCALITY_BY_REGION_LOCATION,
+ SNAPSHOT_INPUTFORMAT_LOCALITY_BY_REGION_LOCATION_DEFAULT);
+
+Connection connection = null;
+RegionLocator regionLocator = null;
+if (localityEnabled && useRegionLoc) {
+ Configuration newConf = new Configuration(conf);
+ newConf.setInt("hbase.hconnection.threads.max", 1);
+ try {
+connection = ConnectionFactory.createConnection(newConf);
+regionLocator = connection.getRegionLocator(htd.getTableName());
+
+/* Get all locations for the table and cache it */
+regionLocator.getAllRegionLocations();
+ } finally {
+if (connection != null) {
+ connection.close();
+}
+ }
+}
+
List splits = new ArrayList<>();
for (RegionInfo hri : regionManifests) {
// load region descriptor
+ List hosts = null;
+ if (localityEnabled) {
+if (regionLocator != null) {
+ /* Get Location from the local cache */
+ HRegionLocation
+location = regionLocator.getRegionLocation(hri.getStartKey(),
false);
+
+ hosts = new ArrayList<>(1);
+ hosts.add(location.getHostname());
+} else {
+ hosts = calculateLocationsForInputSplit(conf, htd, hri, tableDir);
+}
+ }
if (numSplits > 1) {
byte[][] sp = sa.split(hri.getStartKey(), hri.getEndKey(), numSplits,
true);
for (int i = 0; i < sp.length - 1; i++) {
if (PrivateCellUtil.overlappingKeys(scan.getStartRow(),
scan.getStopRow(), sp[i],
sp[i + 1])) {
-List hosts =
-calculateLocationsForInputSplit(conf, htd, hri, tableDir,
localityEnabled);
Scan boundedScan = new Scan(scan);
if (scan.getStartRow().length == 0) {
@@ -425,8 +470,7 @@ public class TableSnapshotInputFormatImpl {
} else {
[hbase] 02/03: HBASE-25988 Store the store file list by a file (#3578)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-26067
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit f642ba3409814b0fdab34c5b971a5bea7f4b3e15
Author: Duo Zhang
AuthorDate: Thu Aug 26 18:51:12 2021 +0800
HBASE-25988 Store the store file list by a file (#3578)
Signed-off-by: Wellington Chevreuil
---
.../protobuf/server/region/StoreFileTracker.proto | 29 +++--
.../hadoop/hbase/regionserver/StoreContext.java| 5 +
.../hadoop/hbase/regionserver/StoreEngine.java | 8 +-
.../storefiletracker/DefaultStoreFileTracker.java | 5 +-
.../FileBasedStoreFileTracker.java | 142 +
.../storefiletracker/StoreFileListFile.java| 142 +
.../storefiletracker/StoreFileTrackerBase.java | 11 +-
.../storefiletracker/StoreFileTrackerFactory.java | 12 +-
.../TestRegionWithFileBasedStoreFileTracker.java | 109
9 files changed, 430 insertions(+), 33 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java
b/hbase-protocol-shaded/src/main/protobuf/server/region/StoreFileTracker.proto
similarity index 57%
copy from
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java
copy to
hbase-protocol-shaded/src/main/protobuf/server/region/StoreFileTracker.proto
index 4f7231b..2a269ea 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java
+++
b/hbase-protocol-shaded/src/main/protobuf/server/region/StoreFileTracker.proto
@@ -15,21 +15,22 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.hbase.regionserver.storefiletracker;
+syntax = "proto2";
+// This file contains protocol buffers that are used for store file tracker.
+package hbase.pb;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.regionserver.StoreContext;
-import org.apache.yetus.audience.InterfaceAudience;
+option java_package = "org.apache.hadoop.hbase.shaded.protobuf.generated";
+option java_outer_classname = "StoreFileTrackerProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
-/**
- * Factory method for creating store file tracker.
- */
-@InterfaceAudience.Private
-public final class StoreFileTrackerFactory {
+message StoreFileEntry {
+ required string name = 1;
+ required uint64 size = 2;
+}
- public static StoreFileTracker create(Configuration conf, TableName
tableName,
-boolean isPrimaryReplica, StoreContext ctx) {
-return new DefaultStoreFileTracker(conf, tableName, isPrimaryReplica, ctx);
- }
+message StoreFileList {
+ required uint64 timestamp = 1;
+ repeated StoreFileEntry store_file = 2;
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java
index 2a9f968..588f8f4 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java
@@ -22,6 +22,7 @@ import java.util.Collection;
import java.util.function.Supplier;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.CellComparator;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.io.HeapSize;
@@ -109,6 +110,10 @@ public final class StoreContext implements HeapSize {
return coprocessorHost;
}
+ public TableName getTableName() {
+return getRegionInfo().getTable();
+ }
+
public RegionInfo getRegionInfo() {
return regionFileSystem.getRegionInfo();
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java
index f9d6c29..61b7dfa 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java
@@ -172,9 +172,9 @@ public abstract class StoreEnginehttp://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver.storefiletracker;
+
+import java.io.IOException;
+import ja
[hbase] 03/03: HBASE-26079 Use StoreFileTracker when splitting and merging (#3617)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-26067
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 0c8db602a946cc9fdbb12893d27eb3a2b0c23d23
Author: Wellington Ramos Chevreuil
AuthorDate: Wed Sep 8 10:31:49 2021 +0100
HBASE-26079 Use StoreFileTracker when splitting and merging (#3617)
Signed-off-by: Duo Zhang
---
.../assignment/MergeTableRegionsProcedure.java | 23 +-
.../assignment/SplitTableRegionProcedure.java | 41 ++--
.../hbase/regionserver/HRegionFileSystem.java | 42 +++-
.../storefiletracker/DefaultStoreFileTracker.java | 4 +-
.../storefiletracker/StoreFileTracker.java | 1 -
.../storefiletracker/StoreFileTrackerFactory.java | 33 ++-
.../hbase/regionserver/TestDefaultStoreEngine.java | 1 +
.../regionserver/TestDirectStoreSplitsMerges.java | 32 ++-
.../hadoop/hbase/regionserver/TestHStoreFile.java | 19 +-
.../regionserver/TestMergesSplitsAddToTracker.java | 262 +
.../hbase/regionserver/TestStripeStoreEngine.java | 1 +
.../storefiletracker/TestStoreFileTracker.java}| 42 ++--
12 files changed, 436 insertions(+), 65 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
index fb57cb9..4b25a28 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
@@ -24,6 +24,8 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.stream.Stream;
+
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.MetaMutationAnnotation;
@@ -53,6 +55,8 @@ import org.apache.hadoop.hbase.quotas.QuotaExceededException;
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.HStoreFile;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
+import
org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.hbase.wal.WALSplitUtil;
@@ -584,40 +588,47 @@ public class MergeTableRegionsProcedure
final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
final Path tableDir = CommonFSUtils.getTableDir(mfs.getRootDir(),
regionsToMerge[0].getTable());
final FileSystem fs = mfs.getFileSystem();
-
+List mergedFiles = new ArrayList<>();
HRegionFileSystem mergeRegionFs =
HRegionFileSystem.createRegionOnFileSystem(
env.getMasterConfiguration(), fs, tableDir, mergedRegion);
for (RegionInfo ri: this.regionsToMerge) {
HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(
env.getMasterConfiguration(), fs, tableDir, ri, false);
- mergeStoreFiles(env, regionFs, mergeRegionFs, mergedRegion);
+ mergedFiles.addAll(mergeStoreFiles(env, regionFs, mergeRegionFs,
mergedRegion));
}
assert mergeRegionFs != null;
-mergeRegionFs.commitMergedRegion();
+mergeRegionFs.commitMergedRegion(mergedFiles, env);
// Prepare to create merged regions
env.getAssignmentManager().getRegionStates().
getOrCreateRegionStateNode(mergedRegion).setState(State.MERGING_NEW);
}
- private void mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem
regionFs,
+ private List mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem
regionFs,
HRegionFileSystem mergeRegionFs, RegionInfo mergedRegion) throws
IOException {
final TableDescriptor htd = env.getMasterServices().getTableDescriptors()
.get(mergedRegion.getTable());
+List mergedFiles = new ArrayList<>();
for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {
String family = hcd.getNameAsString();
- final Collection storeFiles =
regionFs.getStoreFiles(family);
+ Configuration trackerConfig =
+
StoreFileTrackerFactory.mergeConfigurations(env.getMasterConfiguration(), htd,
hcd);
+ StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig,
true,
+family, regionFs);
+ final Collection storeFiles = tracker.load();
if (storeFiles != null && storeFiles.size() > 0) {
for (StoreFileInfo storeFileInfo : storeFiles) {
// Create reference file(s) to parent region file here in mergedDir.
// As this procedure is running on master, use CacheConfig.DISABLED
means
// don't cache any block.
- mergeRegionFs.mergeStoreFi
[hbase] 01/03: HBASE-26064 Introduce a StoreFileTracker to abstract the store file tracking logic
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-26067
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit df87f53771faad1a28296e2b97f3343cf9a6e968
Author: Duo Zhang
AuthorDate: Thu Jul 29 18:35:19 2021 +0800
HBASE-26064 Introduce a StoreFileTracker to abstract the store file
tracking logic
Signed-off-by: Wellington Chevreuil
---
.../hadoop/hbase/mob/DefaultMobStoreCompactor.java | 22 +-
.../hadoop/hbase/mob/DefaultMobStoreFlusher.java | 4 +-
.../regionserver/CreateStoreFileWriterParams.java | 134
.../hbase/regionserver/DateTieredStoreEngine.java | 5 +-
.../hbase/regionserver/DefaultStoreEngine.java | 5 +-
.../hbase/regionserver/DefaultStoreFlusher.java| 11 +-
.../hadoop/hbase/regionserver/HMobStore.java | 3 +-
.../hbase/regionserver/HRegionFileSystem.java | 10 +-
.../apache/hadoop/hbase/regionserver/HStore.java | 699 +
.../hadoop/hbase/regionserver/StoreContext.java| 9 +
.../hadoop/hbase/regionserver/StoreEngine.java | 438 -
.../hbase/regionserver/StoreFileManager.java | 9 +
.../hadoop/hbase/regionserver/StoreFlusher.java| 9 +-
.../hadoop/hbase/regionserver/StoreUtils.java | 36 +-
.../hbase/regionserver/StripeStoreEngine.java | 9 +-
.../hbase/regionserver/StripeStoreFlusher.java | 9 +-
.../compactions/AbstractMultiOutputCompactor.java | 7 +-
.../hbase/regionserver/compactions/Compactor.java | 36 +-
.../regionserver/compactions/DefaultCompactor.java | 16 +-
.../storefiletracker/DefaultStoreFileTracker.java | 61 ++
.../storefiletracker/StoreFileTracker.java | 75 +++
.../storefiletracker/StoreFileTrackerBase.java | 178 ++
.../storefiletracker/StoreFileTrackerFactory.java | 35 ++
.../util/compaction/MajorCompactionRequest.java| 1 -
.../org/apache/hadoop/hbase/TestIOFencing.java | 12 +-
.../regionserver/TestCacheOnWriteInSchema.java | 6 +-
.../hbase/regionserver/TestDefaultStoreEngine.java | 5 +-
.../hadoop/hbase/regionserver/TestHRegion.java | 4 +-
.../hadoop/hbase/regionserver/TestHStore.java | 28 +-
.../TestRegionMergeTransactionOnCluster.java | 6 +-
.../regionserver/TestStoreFileRefresherChore.java | 3 +-
.../regionserver/TestStoreScannerClosure.java | 6 +-
.../hbase/regionserver/TestStripeStoreEngine.java | 2 +
.../compactions/TestDateTieredCompactor.java | 12 +-
.../compactions/TestStripeCompactionPolicy.java| 12 +-
.../compactions/TestStripeCompactor.java | 12 +-
36 files changed, 1224 insertions(+), 705 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
index c45fdff..01fe000 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
@@ -29,7 +29,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map.Entry;
import java.util.Optional;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -144,17 +143,16 @@ public class DefaultMobStoreCompactor extends
DefaultCompactor {
};
private final CellSinkFactory writerFactory =
- new CellSinkFactory() {
-@Override
-public StoreFileWriter createWriter(InternalScanner scanner,
-
org.apache.hadoop.hbase.regionserver.compactions.Compactor.FileDetails fd,
-boolean shouldDropBehind, boolean major) throws IOException {
- // make this writer with tags always because of possible new cells
with tags.
- return store.createWriterInTmp(fd.maxKeyCount,
-major ? majorCompactionCompression : minorCompactionCompression,
-true, true, true, shouldDropBehind);
-}
- };
+new CellSinkFactory() {
+ @Override
+ public StoreFileWriter createWriter(InternalScanner scanner,
+org.apache.hadoop.hbase.regionserver.compactions.Compactor.FileDetails
fd,
+boolean shouldDropBehind, boolean major) throws IOException {
+// make this writer with tags always because of possible new cells
with tags.
+return store.getStoreEngine().createWriter(
+ createParams(fd, shouldDropBehind,
major).includeMVCCReadpoint(true).includesTag(true));
+ }
+};
public DefaultMobStoreCompactor(Configuration conf, HStore store) {
super(conf, store);
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java
index 480b85c..4a1dc7b 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/D
[hbase] branch HBASE-26067 updated (ff2995c -> 0c8db60)
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a change to branch HBASE-26067 in repository https://gitbox.apache.org/repos/asf/hbase.git. discard ff2995c HBASE-26079 Use StoreFileTracker when splitting and merging (#3617) omit 488c21e HBASE-25988 Store the store file list by a file (#3578) omit 1848381 HBASE-26064 Introduce a StoreFileTracker to abstract the store file tracking logic add 11b32db HBASE-26227 Forward port HBASE-26223 test code to branch-2.4+ (#3632) add c770b1d HBASE-26189 Reduce log level of CompactionProgress notice to DEBUG (#3579) add 7cc876d Revert "HBASE-26189 Reduce log level of CompactionProgress notice to DEBUG (#3579)" add 07c0eee HBASE-26189 Reduce log level of CompactionProgress notice to DEBUG (#3579) add 0098164 HBASE-26216 Move HRegionServer.abort(String) to Abortable as a default method (#3642) add c4daabd HBASE-26232 SEEK_NEXT_USING_HINT is ignored on reversed Scans (#3639) add 889049e HBASE-26235 We could start RegionServerTracker before becoming active master (#3645) add a0864ed HBASE-26103 Deprecate BufferedMutatorParams#pool method (#3635) add d19728b HBASE-25642 Fix or stop warning about already cached block (#3638) add ed56a40 HBASE-25588 Excessive logging of "hbase.zookeeper.useMulti is deprecated. Default to true always." (#3640) add 3c64f86 HBASE-26244 Avoid trim the error stack trace when running UT with maven (#3654) add 36884c3 HBASE-26210 HBase Write should be doomed to hang when cell size exceeds InmemoryFlushSize for CompactingMemStore (#3604) add 345d725 HBASE-25773 TestSnapshotScannerHDFSAclController.setupBeforeClass is flaky (#3651) add f62caa5 HBASE-26205 Use specified cluster conf for UserProvider in TableMRUtil#initCredentialsForCluster (#3592) add a15e94a HBASE-26147 Add a dry run mode to the balancer, where moves are calculated but not actually executed add f022692 HBASE-26157 Expose some IA.LimitedPrivate interface in TestingHBaseCluster (#3643) add ec747bc HBASE-26106 AbstractFSWALProvider#getArchivedLogPath doesn't look for wal file in all oldWALs directory. (#3636) add dd293c8 HBASE-26230 Start an in process HRegionServer in maintenance mode (#3659) add afbdd41 HBASE-26237 Improve computation complexity for primaryRegionCountSkewCostFunction (#3657) add b7c6a0b HBASE-26228 updateRSGroupConfig operation should be authorized by AccessController (#3633) add 03b6f2d HBASE-26163 Better logging in RSGroupInfoManagerImpl (#3610) add fae6261 HBASE-26197 Fix some obvious bugs in MultiByteBuff.put (#3586) new df87f53 HBASE-26064 Introduce a StoreFileTracker to abstract the store file tracking logic new f642ba3 HBASE-25988 Store the store file list by a file (#3578) new 0c8db60 HBASE-26079 Use StoreFileTracker when splitting and merging (#3617) This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (ff2995c) \ N -- N -- N refs/heads/HBASE-26067 (0c8db60) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. The 3 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../PrimaryRegionCountSkewCostFunction.java| 10 +- .../java/org/apache/hadoop/hbase/Abortable.java| 10 + .../java/org/apache/hadoop/hbase/client/Admin.java | 40 ++- .../hadoop/hbase/client/AdminOverAsyncAdmin.java | 9 +- .../org/apache/hadoop/hbase/client/AsyncAdmin.java | 37 ++- .../hadoop/hbase/client/AsyncHBaseAdmin.java | 8 +- .../apache/hadoop/hbase/client/BalanceRequest.java | 114 + .../hadoop/hbase/client/BalanceResponse.java | 126 ++ .../hadoop/hbase/client/BufferedMutatorParams.java | 8 + .../hadoop/hbase/client/RawAsyncHBaseAdmin.java| 29 ++- .../hadoop/hbase/shaded/protobuf/ProtobufUtil.java | 63 + .../hbase/shaded/protobuf/RequestConverter.java| 10 - .../apache/hadoop/hbase/zookeeper/ZNodePaths.java | 5 + .../org/apache/hadoop/hbase/nio/MultiByteBuff.java | 119 +++-- .../apache/hadoop/hbase/nio/TestMultiByteBuff.java | 112 + .../hadoop/hbase/mapreduce/TableMapReduceUtil.java | 2 +- .../hadoop/hbase/mapreduce/WALInputFormat.java | 5 +- .../hbase/
[hbase] branch HBASE-26067 updated: HBASE-26079 Use StoreFileTracker when splitting and merging (#3617)
This is an automated email from the ASF dual-hosted git repository.
wchevreuil pushed a commit to branch HBASE-26067
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/HBASE-26067 by this push:
new ff2995c HBASE-26079 Use StoreFileTracker when splitting and merging
(#3617)
ff2995c is described below
commit ff2995cca2650380f76ea4b3ad71489847b30019
Author: Wellington Ramos Chevreuil
AuthorDate: Wed Sep 8 10:31:49 2021 +0100
HBASE-26079 Use StoreFileTracker when splitting and merging (#3617)
Signed-off-by: Duo Zhang
---
.../assignment/MergeTableRegionsProcedure.java | 23 +-
.../assignment/SplitTableRegionProcedure.java | 41 ++--
.../hbase/regionserver/HRegionFileSystem.java | 42 +++-
.../storefiletracker/DefaultStoreFileTracker.java | 4 +-
.../storefiletracker/StoreFileTracker.java | 1 -
.../storefiletracker/StoreFileTrackerFactory.java | 33 ++-
.../hbase/regionserver/TestDefaultStoreEngine.java | 1 +
.../regionserver/TestDirectStoreSplitsMerges.java | 32 ++-
.../hadoop/hbase/regionserver/TestHStoreFile.java | 19 +-
.../regionserver/TestMergesSplitsAddToTracker.java | 262 +
.../hbase/regionserver/TestStripeStoreEngine.java | 1 +
.../storefiletracker/TestStoreFileTracker.java}| 42 ++--
12 files changed, 436 insertions(+), 65 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
index fb57cb9..4b25a28 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
@@ -24,6 +24,8 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.stream.Stream;
+
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.MetaMutationAnnotation;
@@ -53,6 +55,8 @@ import org.apache.hadoop.hbase.quotas.QuotaExceededException;
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.HStoreFile;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
+import
org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.hbase.wal.WALSplitUtil;
@@ -584,40 +588,47 @@ public class MergeTableRegionsProcedure
final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
final Path tableDir = CommonFSUtils.getTableDir(mfs.getRootDir(),
regionsToMerge[0].getTable());
final FileSystem fs = mfs.getFileSystem();
-
+List mergedFiles = new ArrayList<>();
HRegionFileSystem mergeRegionFs =
HRegionFileSystem.createRegionOnFileSystem(
env.getMasterConfiguration(), fs, tableDir, mergedRegion);
for (RegionInfo ri: this.regionsToMerge) {
HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(
env.getMasterConfiguration(), fs, tableDir, ri, false);
- mergeStoreFiles(env, regionFs, mergeRegionFs, mergedRegion);
+ mergedFiles.addAll(mergeStoreFiles(env, regionFs, mergeRegionFs,
mergedRegion));
}
assert mergeRegionFs != null;
-mergeRegionFs.commitMergedRegion();
+mergeRegionFs.commitMergedRegion(mergedFiles, env);
// Prepare to create merged regions
env.getAssignmentManager().getRegionStates().
getOrCreateRegionStateNode(mergedRegion).setState(State.MERGING_NEW);
}
- private void mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem
regionFs,
+ private List mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem
regionFs,
HRegionFileSystem mergeRegionFs, RegionInfo mergedRegion) throws
IOException {
final TableDescriptor htd = env.getMasterServices().getTableDescriptors()
.get(mergedRegion.getTable());
+List mergedFiles = new ArrayList<>();
for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {
String family = hcd.getNameAsString();
- final Collection storeFiles =
regionFs.getStoreFiles(family);
+ Configuration trackerConfig =
+
StoreFileTrackerFactory.mergeConfigurations(env.getMasterConfiguration(), htd,
hcd);
+ StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig,
true,
+family, regionFs);
+ final Collection storeFiles = tracker.load();
if (storeFiles != null && storeFiles.size() > 0) {
for (StoreFileInfo storeFileInfo : storeFiles) {
// Create reference file(s) to paren
