subject:"\[GitHub\] \[hbase\] saintstack commented on a change in pull request #1746\: HBASE\-24388 Introduce a 'local root region' at master side to store t…"

[GitHub] [hbase] saintstack commented on a change in pull request #1746: HBASE-24388 Introduce a 'local root region' at master side to store t…

2020-05-21 Thread GitBox



saintstack commented on a change in pull request #1746:
URL: https://github.com/apache/hbase/pull/1746#discussion_r428743594



##
File path: 
hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
##
@@ -552,4 +553,6 @@ default SplitWALManager getSplitWALManager(){
* @return The state of the load balancer, or false if the load balancer 
isn't defined.
*/
   boolean isBalancerOn();
+
+  RootTable getRootTable();

Review comment:
   If it the only consumer... yeah, maybe. No hurry.





This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org

[GitHub] [hbase] saintstack commented on a change in pull request #1746: HBASE-24388 Introduce a 'local root region' at master side to store t…

2020-05-21 Thread GitBox



saintstack commented on a change in pull request #1746:
URL: https://github.com/apache/hbase/pull/1746#discussion_r428743081



##
File path: 
hbase-server/src/main/java/org/apache/hadoop/hbase/master/root/RootTable.java
##
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.root;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.master.cleaner.DirScanPool;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil;
+import org.apache.hadoop.hbase.region.LocalRegion;
+import org.apache.hadoop.hbase.region.LocalRegionParams;
+import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Used to store the location of meta region.
+ */
+@InterfaceAudience.Private
+public class RootTable {
+
+  private static final Logger LOG = LoggerFactory.getLogger(RootTable.class);
+
+  static final String MAX_WALS_KEY = "hbase.root.table.region.maxwals";
+
+  private static final int DEFAULT_MAX_WALS = 10;
+
+  static final String USE_HSYNC_KEY = "hbase.root.table.region.wal.hsync";
+
+  static final String RING_BUFFER_SLOT_COUNT = 
"hbase.root.table.region.maxwals";
+
+  private static final int DEFAULT_RING_BUFFER_SLOT_COUNT = 64;
+
+  static final String ROOT_TABLE_DIR = "RootTable";
+
+  static final String HFILECLEANER_PLUGINS = 
"hbase.root.table.region.hfilecleaner.plugins";
+
+  static final String FLUSH_SIZE_KEY = "hbase.root.table.region.flush.size";
+
+  static final long DEFAULT_FLUSH_SIZE = 
TableDescriptorBuilder.DEFAULT_MEMSTORE_FLUSH_SIZE;
+
+  static final String FLUSH_PER_CHANGES_KEY = 
"hbase.root.table.region.flush.per.changes";
+
+  private static final long DEFAULT_FLUSH_PER_CHANGES = 1_000_000;
+
+  static final String FLUSH_INTERVAL_MS_KEY = 
"hbase.root.table.region.flush.interval.ms";
+
+  // default to flush every 15 minutes, for safety
+  private static final long DEFAULT_FLUSH_INTERVAL_MS = 
TimeUnit.MINUTES.toMillis(15);
+
+  static final String COMPACT_MIN_KEY = "hbase.root.table.region.compact.min";
+
+  private static final int DEFAULT_COMPACT_MIN = 4;
+
+  static final String ROLL_PERIOD_MS_KEY = 
"hbase.root.table.region.walroll.period.ms";
+
+  private static final long DEFAULT_ROLL_PERIOD_MS = 
TimeUnit.MINUTES.toMillis(15);
+
+  static final TableName TABLE_NAME = TableName.valueOf("master:root");
+
+  private static final TableDescriptor TABLE_DESC = 
TableDescriptorBuilder.newBuilder(TABLE_NAME)
+
.setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(HConstants.CATALOG_FAMILY)
+  .setMaxVersions(HConstants.DEFAULT_HBASE_META_VERSIONS).setInMemory(true)
+  
.setBlocksize(HConstants.DEFAULT_HBASE_META_BLOCK_SIZE).setBloomFilterType(BloomType.ROWCOL)
+  .setDataBlockEncoding(DataBlockEncoding.ROW_INDEX_V1).build())
+.build();
+
+  private final Server server;
+
+  private final DirScanPool cleanerPool;
+
+  private LocalRegion region;
+
+  public RootTable(Server server, DirScanPool cleanerPool) {
+this.server = server;
+this.cleanerPool = cleanerPool;
+  }
+
+  public void initialize() throws IOException {
+LOG.info("Initializing root table...");
+LocalRegionParams params = new 
LocalRegionParams().server(server).regionDirName(ROOT_TABLE_DIR)

Review comment:
   Yeah, would be good if a general location for master local store.





This is an automated message fro

[GitHub] [hbase] saintstack commented on a change in pull request #1746: HBASE-24388 Introduce a 'local root region' at master side to store t…

2020-05-21 Thread GitBox



saintstack commented on a change in pull request #1746:
URL: https://github.com/apache/hbase/pull/1746#discussion_r428740739



##
File path: 
hbase-server/src/main/java/org/apache/hadoop/hbase/region/LocalRegion.java
##
@@ -0,0 +1,328 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.region;
+
+import static org.apache.hadoop.hbase.HConstants.HREGION_LOGDIR_NAME;
+
+import java.io.IOException;
+import java.util.Collections;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseIOException;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RegionInfoBuilder;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegion.FlushResult;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.CommonFSUtils;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
+import org.apache.hadoop.hbase.util.RecoverLeaseFSUtils;
+import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
+import org.apache.hadoop.hbase.wal.WAL;
+import org.apache.hadoop.hbase.wal.WALFactory;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import 
org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
+import org.apache.hbase.thirdparty.com.google.common.math.IntMath;
+
+/**
+ * A region that stores data in a separated directory on WAL file system.

Review comment:
   Pardon me. 'snowflakes' are purportedly unique; no two are alike. 
'snowflaking' is making something 'special', a one-off. I was asking if we need 
to do the special trick where this local region runs on the WAL FS exclusively 
making it different to how other Regions do their storage spread across FS's... 
one for data and another for WAL.





This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org

[GitHub] [hbase] saintstack commented on a change in pull request #1746: HBASE-24388 Introduce a 'local root region' at master side to store t…

2020-05-20 Thread GitBox



saintstack commented on a change in pull request #1746:
URL: https://github.com/apache/hbase/pull/1746#discussion_r428196462



##
File path: 
hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
##
@@ -866,6 +877,50 @@ protected AssignmentManager 
createAssignmentManager(MasterServices master) {
 return new AssignmentManager(master);
   }
 
+  private void createRootTable() throws IOException, KeeperException {
+RootTable rootTable = new RootTable(this, cleanerPool);
+rootTable.initialize();
+// try migrate data from zookeeper
+try (RegionScanner scanner =
+  rootTable.getScanner(new Scan().addFamily(HConstants.CATALOG_FAMILY))) {
+  List cells = new ArrayList<>();
+  boolean moreRows = scanner.next(cells);
+  if (!cells.isEmpty() || moreRows) {
+// notice that all replicas for a region are in the same row, so the 
migration can be
+// done with in a one row put, which means if we have data in root 
table then we can make
+// sure that the migration is done.

Review comment:
   Good

##
File path: 
hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
##
@@ -866,6 +877,50 @@ protected AssignmentManager 
createAssignmentManager(MasterServices master) {
 return new AssignmentManager(master);
   }
 
+  private void createRootTable() throws IOException, KeeperException {

Review comment:
   Would be cool if this was not inline in HMaster class. It too big as it 
is.

##
File path: 
hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
##
@@ -866,6 +877,50 @@ protected AssignmentManager 
createAssignmentManager(MasterServices master) {
 return new AssignmentManager(master);
   }
 
+  private void createRootTable() throws IOException, KeeperException {
+RootTable rootTable = new RootTable(this, cleanerPool);
+rootTable.initialize();
+// try migrate data from zookeeper
+try (RegionScanner scanner =
+  rootTable.getScanner(new Scan().addFamily(HConstants.CATALOG_FAMILY))) {
+  List cells = new ArrayList<>();
+  boolean moreRows = scanner.next(cells);
+  if (!cells.isEmpty() || moreRows) {
+// notice that all replicas for a region are in the same row, so the 
migration can be
+// done with in a one row put, which means if we have data in root 
table then we can make
+// sure that the migration is done.
+LOG.info("Root table already has data in it, skip migrating...");
+this.rootTable = rootTable;
+return;
+  }
+}
+// start migrating
+byte[] row = 
MetaTableAccessor.getMetaKeyForRegion(RegionInfoBuilder.FIRST_META_REGIONINFO);
+Put put = new Put(row);
+List metaReplicaNodes = zooKeeper.getMetaReplicaNodes();
+StringBuilder info = new StringBuilder("Migrating meta location:");
+for (String metaReplicaNode : metaReplicaNodes) {
+  int replicaId = 
zooKeeper.getZNodePaths().getMetaReplicaIdFromZnode(metaReplicaNode);
+  RegionState state = MetaTableLocator.getMetaRegionState(zooKeeper, 
replicaId);
+  info.append(" ").append(state);
+  put.setTimestamp(state.getStamp());
+  MetaTableAccessor.addRegionInfo(put, state.getRegion());
+  if (state.getServerName() != null) {
+MetaTableAccessor.addLocation(put, state.getServerName(), 
HConstants.NO_SEQNUM, replicaId);
+  }
+  
put.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(put.getRow())
+.setFamily(HConstants.CATALOG_FAMILY)
+
.setQualifier(RegionStateStore.getStateColumn(replicaId)).setTimestamp(put.getTimestamp())
+
.setType(Cell.Type.Put).setValue(Bytes.toBytes(state.getState().name())).build());
+}
+if (!put.isEmpty()) {
+  LOG.info(info.toString());
+} else {
+  LOG.info("No meta location avaiable on zookeeper, skip migrating...");
+}
+this.rootTable = rootTable;
+  }
+

Review comment:
   good

##
File path: 
hbase-server/src/main/java/org/apache/hadoop/hbase/region/LocalRegion.java
##
@@ -0,0 +1,328 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.region;

[GitHub] [hbase] saintstack commented on a change in pull request #1746: HBASE-24388 Introduce a 'local root region' at master side to store t…

[GitHub] [hbase] saintstack commented on a change in pull request #1746: HBASE-24388 Introduce a 'local root region' at master side to store t…

[GitHub] [hbase] saintstack commented on a change in pull request #1746: HBASE-24388 Introduce a 'local root region' at master side to store t…

[GitHub] [hbase] saintstack commented on a change in pull request #1746: HBASE-24388 Introduce a 'local root region' at master side to store t…

4 matches

Site Navigation

Mail list logo

Footer information