This is an automated email from the ASF dual-hosted git repository.
niuyulin pushed a commit to branch HBASE-25714
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/HBASE-25714 by this push:
new 74fbc1b HBASE-26085 Support MOB when do compaction offload (#3479)
74fbc1b is described below
commit 74fbc1bcc494686b86564e2b5846577f2ddf93eb
Author: niuyulin <[email protected]>
AuthorDate: Thu Jul 15 14:19:54 2021 +0800
HBASE-26085 Support MOB when do compaction offload (#3479)
Signed-off-by: Duo Zhang <[email protected]>
---
.../compactionserver/CompactionThreadManager.java | 11 ++-
.../compactionserver/TestMobCompactionOffload.java | 87 ++++++++++++++++++++++
.../hbase/mob/TestMobCompactionWithDefaults.java | 16 ++--
3 files changed, 107 insertions(+), 7 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/compactionserver/CompactionThreadManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/compactionserver/CompactionThreadManager.java
index 2a04e9e..bd78cc4 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/compactionserver/CompactionThreadManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/compactionserver/CompactionThreadManager.java
@@ -41,9 +41,11 @@ import org.apache.hadoop.hbase.client.AsyncRegionServerAdmin;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.mob.MobFileCache;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.monitoring.TaskMonitor;
import org.apache.hadoop.hbase.regionserver.CompactThreadControl;
+import org.apache.hadoop.hbase.regionserver.HMobStore;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.HStore;
@@ -360,7 +362,14 @@ public class CompactionThreadManager implements
ThroughputControllerService {
HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs,
CommonFSUtils.getTableDir(rootDir, htd.getTableName()), hri);
HRegion region = new HRegion(regionFs, null, conf, htd, null);
- HStore store = new HStore(region,
htd.getColumnFamily(Bytes.toBytes(familyName)), conf, false);
+ ColumnFamilyDescriptor columnFamilyDescriptor =
htd.getColumnFamily(Bytes.toBytes(familyName));
+ HStore store;
+ if (columnFamilyDescriptor.isMobEnabled()) {
+ region.setMobFileCache(new MobFileCache(conf));
+ store = new HMobStore(region, columnFamilyDescriptor, conf, false);
+ } else {
+ store = new HStore(region, columnFamilyDescriptor, conf, false);
+ }
OptionalLong maxSequenceId = store.getMaxSequenceId();
LOG.info("store max sequence id: {}", maxSequenceId.orElse(0));
region.getMVCC().advanceTo(maxSequenceId.orElse(0));
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/compactionserver/TestMobCompactionOffload.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/compactionserver/TestMobCompactionOffload.java
new file mode 100644
index 0000000..ebd605f
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/compactionserver/TestMobCompactionOffload.java
@@ -0,0 +1,87 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.compactionserver;
+
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.StartMiniClusterOption;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.mob.MobFileCleanerChore;
+import org.apache.hadoop.hbase.mob.TestMobCompactionWithDefaults;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.RegionSplitter;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.experimental.categories.Category;
+
+@Category(LargeTests.class)
+public class TestMobCompactionOffload extends TestMobCompactionWithDefaults {
+ private static HCompactionServer COMPACTION_SERVER;
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestMobCompactionOffload.class);
+
+ @BeforeClass
+ public static void htuStart() throws Exception {
+ HTU = new HBaseTestingUtility();
+ conf = HTU.getConfiguration();
+ setMobTestConf();
+
HTU.startMiniCluster(StartMiniClusterOption.builder().numCompactionServers(1).build());
+ HTU.getAdmin().switchCompactionOffload(true);
+ HTU.getMiniHBaseCluster().waitForActiveAndReadyMaster();
+ COMPACTION_SERVER =
HTU.getMiniHBaseCluster().getCompactionServerThreads().get(0)
+ .getCompactionServer();
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ admin = HTU.getAdmin();
+ cleanerChore = new MobFileCleanerChore();
+ familyDescriptor =
ColumnFamilyDescriptorBuilder.newBuilder(fam).setMobEnabled(true)
+ .setMobThreshold(mobLen).setMaxVersions(1).build();
+ tableDescriptor = HTU.createModifyableTableDescriptor(test.getMethodName())
+
.setColumnFamily(familyDescriptor).setCompactionOffloadEnabled(true).build();
+ RegionSplitter.UniformSplit splitAlgo = new RegionSplitter.UniformSplit();
+ byte[][] splitKeys = splitAlgo.split(numRegions);
+ table = HTU.createTable(tableDescriptor, splitKeys).getName();
+ COMPACTION_SERVER.requestCount.reset();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ // ensure do compaction on compaction server
+ HTU.waitFor(6000, () -> COMPACTION_SERVER.requestCount.sum() > 0);
+ admin.disableTable(tableDescriptor.getTableName());
+ admin.deleteTable(tableDescriptor.getTableName());
+ }
+
+ @Override
+ protected void waitUntilCompactionIsComplete(TableName table) {
+ while
(COMPACTION_SERVER.compactionThreadManager.getRunningCompactionTasks().size() >
0) {
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionWithDefaults.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionWithDefaults.java
index 149de20..eacd8a0 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionWithDefaults.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionWithDefaults.java
@@ -95,7 +95,7 @@ public class TestMobCompactionWithDefaults {
@Rule
public TestName test = new TestName();
protected TableDescriptor tableDescriptor;
- private ColumnFamilyDescriptor familyDescriptor;
+ protected ColumnFamilyDescriptor familyDescriptor;
protected Admin admin;
protected TableName table = null;
protected int numRegions = 20;
@@ -103,10 +103,7 @@ public class TestMobCompactionWithDefaults {
protected MobFileCleanerChore cleanerChore;
- @BeforeClass
- public static void htuStart() throws Exception {
- HTU = new HBaseTestingUtility();
- conf = HTU.getConfiguration();
+ protected static void setMobTestConf() {
conf.setInt("hfile.format.version", 3);
// Disable automatic MOB compaction
conf.setLong(MobConstants.MOB_COMPACTION_CHORE_PERIOD, 0);
@@ -115,8 +112,15 @@ public class TestMobCompactionWithDefaults {
// Set minimum age to archive to 10 sec
conf.setLong(MobConstants.MIN_AGE_TO_ARCHIVE_KEY, minAgeToArchive);
// Set compacted file discharger interval to a half minAgeToArchive
- conf.setLong("hbase.hfile.compaction.discharger.interval",
minAgeToArchive/2);
+ conf.setLong("hbase.hfile.compaction.discharger.interval", minAgeToArchive
/ 2);
conf.setBoolean("hbase.regionserver.compaction.enabled", false);
+ }
+
+ @BeforeClass
+ public static void htuStart() throws Exception {
+ HTU = new HBaseTestingUtility();
+ conf = HTU.getConfiguration();
+ setMobTestConf();
HTU.startMiniCluster();
}
