Author: tedyu
Date: Wed Jun 22 06:48:28 2011
New Revision: 1138318
URL: http://svn.apache.org/viewvc?rev=1138318&view=rev
Log:
HBASE-4010 HMaster.createTable could be heavily optimized
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java?rev=1138318&r1=1138317&r2=1138318&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
(original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
Wed Jun 22 06:48:28 2011
@@ -47,6 +47,13 @@ import org.apache.hadoop.hbase.catalog.M
public class MetaEditor {
private static final Log LOG = LogFactory.getLog(MetaEditor.class);
+ private static Put makePutFromRegionInfo(HRegionInfo regionInfo) throws
IOException {
+ Put put = new Put(regionInfo.getRegionName());
+ put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
+ Writables.getBytes(regionInfo));
+ return put;
+ }
+
/**
* Adds a META row for the specified new region.
* @param regionInfo region information
@@ -55,15 +62,31 @@ public class MetaEditor {
public static void addRegionToMeta(CatalogTracker catalogTracker,
HRegionInfo regionInfo)
throws IOException {
- Put put = new Put(regionInfo.getRegionName());
- put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
- Writables.getBytes(regionInfo));
catalogTracker.waitForMetaServerConnectionDefault().put(
- CatalogTracker.META_REGION, put);
+ CatalogTracker.META_REGION, makePutFromRegionInfo(regionInfo));
LOG.info("Added region " + regionInfo.getRegionNameAsString() + " to
META");
}
/**
+ * Adds a META row for each of the specified new regions.
+ * @param catalogTracker CatalogTracker
+ * @param regionInfos region information list
+ * @throws IOException if problem connecting or updating meta
+ */
+ public static void addRegionsToMeta(CatalogTracker catalogTracker,
+ List<HRegionInfo> regionInfos)
+ throws IOException {
+ List<Put> puts = new ArrayList<Put>();
+ for (HRegionInfo regionInfo : regionInfos) {
+ puts.add(makePutFromRegionInfo(regionInfo));
+ LOG.debug("Added region " + regionInfo.getRegionNameAsString() + " to
META");
+ }
+ catalogTracker.waitForMetaServerConnectionDefault().put(
+ CatalogTracker.META_REGION, puts);
+ LOG.info("Added " + puts.size() + " regions to META");
+ }
+
+ /**
* Offline parent in meta.
* Used when splitting.
* @param catalogTracker
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1138318&r1=1138317&r2=1138318&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
(original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Wed
Jun 22 06:48:28 2011
@@ -1,5 +1,5 @@
/**
- * Copyright 2010 The Apache Software Foundation
+ * Copyright 2011 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -39,7 +39,6 @@ import org.apache.hadoop.hbase.HColumnDe
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HServerLoad;
-
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
@@ -55,8 +54,8 @@ import org.apache.hadoop.hbase.catalog.M
import org.apache.hadoop.hbase.catalog.MetaReader;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.MetaScanner;
-import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType;
import org.apache.hadoop.hbase.ipc.HBaseRPC;
@@ -76,16 +75,17 @@ import org.apache.hadoop.hbase.master.me
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.monitoring.TaskMonitor;
import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
import org.apache.hadoop.hbase.replication.regionserver.Replication;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.InfoServer;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.Sleeper;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.util.VersionInfo;
-import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.zookeeper.ClusterId;
import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
@@ -975,25 +975,39 @@ implements HMasterInterface, HMasterRegi
// do rename to move it into place?
FSUtils.createTableDescriptor(hTableDescriptor, conf);
- for (HRegionInfo newRegion : newRegions) {
- // 1. Set table enabling flag up in zk.
- try {
- assignmentManager.getZKTable().setEnabledTable(tableName);
- } catch (KeeperException e) {
- throw new IOException("Unable to ensure that the table will be" +
+ // 1. Set table enabling flag up in zk.
+ try {
+ assignmentManager.getZKTable().setEnabledTable(tableName);
+ } catch (KeeperException e) {
+ throw new IOException("Unable to ensure that the table will be" +
" enabled because of a ZooKeeper issue", e);
- }
+ }
+ List<HRegionInfo> regionInfos = new ArrayList<HRegionInfo>();
+ final int batchSize =
this.conf.getInt("hbase.master.createtable.batchsize", 100);
+ HLog hlog = null;
+ for (int regionIdx = 0; regionIdx < newRegions.length; regionIdx++) {
+ HRegionInfo newRegion = newRegions[regionIdx];
// 2. Create HRegion
HRegion region = HRegion.createHRegion(newRegion,
- fileSystemManager.getRootDir(), conf, hTableDescriptor);
+ fileSystemManager.getRootDir(), conf, hTableDescriptor, hlog);
+ if (hlog == null) {
+ hlog = region.getLog();
+ }
- // 3. Insert into META
- MetaEditor.addRegionToMeta(catalogTracker, region.getRegionInfo());
+ regionInfos.add(region.getRegionInfo());
+ if (regionIdx % batchSize == 0) {
+ // 3. Insert into META
+ MetaEditor.addRegionsToMeta(catalogTracker, regionInfos);
+ regionInfos.clear();
+ }
// 4. Close the new region to flush to disk. Close log file too.
region.close();
- region.getLog().closeAndDelete();
+ }
+ hlog.closeAndDelete();
+ if (regionInfos.size() > 0) {
+ MetaEditor.addRegionsToMeta(catalogTracker, regionInfos);
}
// 5. Trigger immediate assignment of the regions in round-robin fashion
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1138318&r1=1138317&r2=1138318&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
(original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
Wed Jun 22 06:48:28 2011
@@ -2778,23 +2778,48 @@ public class HRegion implements HeapSize
public static HRegion createHRegion(final HRegionInfo info, final Path
rootDir,
final Configuration conf, final HTableDescriptor hTableDescriptor)
throws IOException {
+ return createHRegion(info, rootDir, conf, hTableDescriptor, null);
+ }
+
+ /**
+ * Convenience method creating new HRegions. Used by createTable.
+ * The {@link HLog} for the created region needs to be closed explicitly.
+ * Use {@link HRegion#getLog()} to get access.
+ *
+ * @param info Info for region to create.
+ * @param rootDir Root directory for HBase instance
+ * @param conf
+ * @param hTableDescriptor
+ * @param hlog shared HLog
+ * @return new HRegion
+ *
+ * @throws IOException
+ */
+ public static HRegion createHRegion(final HRegionInfo info, final Path
rootDir,
+ final Configuration conf,
+ final HTableDescriptor hTableDescriptor,
+ final HLog hlog)
+ throws IOException {
LOG.info("creating HRegion " + info.getTableNameAsString()
- + " HTD == " + hTableDescriptor + " RootDir = " + rootDir +
- " Table name == " + info.getTableNameAsString());
+ + " HTD == " + hTableDescriptor + " RootDir = " + rootDir +
+ " Table name == " + info.getTableNameAsString());
Path tableDir =
HTableDescriptor.getTableDir(rootDir, info.getTableName());
Path regionDir = HRegion.getRegionDir(tableDir, info.getEncodedName());
FileSystem fs = FileSystem.get(conf);
fs.mkdirs(regionDir);
+ HLog effectiveHLog = hlog;
+ if (hlog == null) {
+ effectiveHLog = new HLog(fs, new Path(regionDir,
HConstants.HREGION_LOGDIR_NAME),
+ new Path(regionDir, HConstants.HREGION_OLDLOGDIR_NAME), conf);
+ }
HRegion region = HRegion.newHRegion(tableDir,
- new HLog(fs, new Path(regionDir, HConstants.HREGION_LOGDIR_NAME),
- new Path(regionDir, HConstants.HREGION_OLDLOGDIR_NAME), conf),
- fs, conf, info, hTableDescriptor, null);
+ effectiveHLog, fs, conf, info, hTableDescriptor, null);
region.initialize();
return region;
}
-
+
/**
* Open a Region.
* @param info Info for region to be opened.
