http://git-wip-us.apache.org/repos/asf/hbase/blob/fa852c4c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java index 221c7a4..e5214ca 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java @@ -40,7 +40,6 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.MetaTableAccessor; -import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.ServerLoad; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; @@ -147,13 +146,13 @@ public class RegionStates { private final TableStateManager tableStateManager; private final RegionStateStore regionStateStore; private final ServerManager serverManager; - private final Server server; + private final MasterServices server; // The maximum time to keep a log split info in region states map static final String LOG_SPLIT_TIME = "hbase.master.maximum.logsplit.keeptime"; static final long DEFAULT_LOG_SPLIT_TIME = 7200000L; // 2 hours - RegionStates(final Server master, final TableStateManager tableStateManager, + RegionStates(final MasterServices master, final TableStateManager tableStateManager, final ServerManager serverManager, final RegionStateStore regionStateStore) { this.tableStateManager = tableStateManager; this.regionStateStore = regionStateStore; @@ -872,7 +871,7 @@ public class RegionStates { private int getRegionReplication(HRegionInfo r) throws IOException { if (tableStateManager != null) { - HTableDescriptor htd = tableStateManager.getTableDescriptors().get(r.getTable()); + HTableDescriptor htd = server.getTableDescriptors().get(r.getTable()); if (htd != null) { return htd.getRegionReplication(); }
http://git-wip-us.apache.org/repos/asf/hbase/blob/fa852c4c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SnapshotOfRegionAssignmentFromMeta.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SnapshotOfRegionAssignmentFromMeta.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SnapshotOfRegionAssignmentFromMeta.java index b03611c..39beba8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SnapshotOfRegionAssignmentFromMeta.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SnapshotOfRegionAssignmentFromMeta.java @@ -141,7 +141,7 @@ public class SnapshotOfRegionAssignmentFromMeta { } }; // Scan hbase:meta to pick up user regions - MetaTableAccessor.fullScan(connection, v); + MetaTableAccessor.fullScanRegions(connection, v); //regionToRegionServerMap = regions; LOG.info("Finished to scan the hbase:meta for the current region assignment" + "snapshot"); http://git-wip-us.apache.org/repos/asf/hbase/blob/fa852c4c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java index d8199ea..5d1e638 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java @@ -17,19 +17,27 @@ */ package org.apache.hadoop.hbase.master; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.Set; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; -import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.TableDescriptor; import org.apache.hadoop.hbase.TableDescriptors; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.client.ClusterConnection; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.TableState; /** @@ -39,24 +47,12 @@ import org.apache.hadoop.hbase.client.TableState; @InterfaceAudience.Private public class TableStateManager { private static final Log LOG = LogFactory.getLog(TableStateManager.class); - private final TableDescriptors descriptors; - private final Map<TableName, TableState.State> tableStates = Maps.newConcurrentMap(); + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + private final MasterServices master; public TableStateManager(MasterServices master) { - this.descriptors = master.getTableDescriptors(); - } - - public void start() throws IOException { - Map<String, TableDescriptor> all = descriptors.getAllDescriptors(); - for (TableDescriptor table : all.values()) { - TableName tableName = table.getHTableDescriptor().getTableName(); - if (LOG.isDebugEnabled()) { - LOG.debug("Adding table state: " + tableName - + ": " + table.getTableState()); - } - tableStates.put(tableName, table.getTableState()); - } + this.master = master; } /** @@ -67,16 +63,13 @@ public class TableStateManager { * @throws IOException */ public void setTableState(TableName tableName, TableState.State newState) throws IOException { - synchronized (tableStates) { - TableDescriptor descriptor = readDescriptor(tableName); - if (descriptor == null) { - throw new TableNotFoundException(tableName); - } - if (descriptor.getTableState() != newState) { - writeDescriptor( - new TableDescriptor(descriptor.getHTableDescriptor(), newState)); - } + lock.writeLock().lock(); + try { + udpateMetaState(tableName, newState); + } finally { + lock.writeLock().unlock(); } + } /** @@ -91,21 +84,23 @@ public class TableStateManager { TableState.State newState, TableState.State... states) throws IOException { - synchronized (tableStates) { - TableDescriptor descriptor = readDescriptor(tableName); - if (descriptor == null) { + lock.writeLock().lock(); + try { + TableState currentState = readMetaState(tableName); + if (currentState == null) { throw new TableNotFoundException(tableName); } - if (TableState.isInStates(descriptor.getTableState(), states)) { - writeDescriptor( - new TableDescriptor(descriptor.getHTableDescriptor(), newState)); + if (currentState.inStates(states)) { + udpateMetaState(tableName, newState); return true; } else { return false; } + } finally { + lock.writeLock().unlock(); } - } + } /** * Set table state to provided but only if table not in specified states @@ -119,42 +114,36 @@ public class TableStateManager { TableState.State newState, TableState.State... states) throws IOException { - synchronized (tableStates) { - TableDescriptor descriptor = readDescriptor(tableName); - if (descriptor == null) { - throw new TableNotFoundException(tableName); - } - if (!TableState.isInStates(descriptor.getTableState(), states)) { - writeDescriptor( - new TableDescriptor(descriptor.getHTableDescriptor(), newState)); - return true; - } else { - return false; - } + TableState currentState = readMetaState(tableName); + if (currentState == null) { + throw new TableNotFoundException(tableName); + } + if (!currentState.inStates(states)) { + udpateMetaState(tableName, newState); + return true; + } else { + return false; } } public boolean isTableState(TableName tableName, TableState.State... states) { - TableState.State tableState = null; try { - tableState = getTableState(tableName); + TableState.State tableState = getTableState(tableName); + return TableState.isInStates(tableState, states); } catch (IOException e) { - LOG.error("Unable to get table state, probably table not exists"); + LOG.error("Unable to get table " + tableName + " state, probably table not exists"); return false; } - return tableState != null && TableState.isInStates(tableState, states); } public void setDeletedTable(TableName tableName) throws IOException { - TableState.State remove = tableStates.remove(tableName); - if (remove == null) { - LOG.warn("Moving table " + tableName + " state to deleted but was " + - "already deleted"); - } + if (tableName.equals(TableName.META_TABLE_NAME)) + return; + MetaTableAccessor.deleteTableState(master.getConnection(), tableName); } public boolean isTablePresent(TableName tableName) throws IOException { - return getTableState(tableName) != null; + return readMetaState(tableName) != null; } /** @@ -164,57 +153,82 @@ public class TableStateManager { * @return tables in given states * @throws IOException */ - public Set<TableName> getTablesInStates(TableState.State... states) throws IOException { - Set<TableName> rv = Sets.newHashSet(); - for (Map.Entry<TableName, TableState.State> entry : tableStates.entrySet()) { - if (TableState.isInStates(entry.getValue(), states)) - rv.add(entry.getKey()); - } + public Set<TableName> getTablesInStates(final TableState.State... states) throws IOException { + final Set<TableName> rv = Sets.newHashSet(); + MetaTableAccessor.fullScanTables(master.getConnection(), new MetaTableAccessor.Visitor() { + @Override + public boolean visit(Result r) throws IOException { + TableState tableState = MetaTableAccessor.getTableState(r); + if (tableState != null && tableState.inStates(states)) + rv.add(tableState.getTableName()); + return true; + } + }); return rv; } + @Nonnull public TableState.State getTableState(TableName tableName) throws IOException { - TableState.State tableState = tableStates.get(tableName); - if (tableState == null) { - TableDescriptor descriptor = readDescriptor(tableName); - if (descriptor != null) - tableState = descriptor.getTableState(); + TableState currentState = readMetaState(tableName); + if (currentState == null) { + throw new TableNotFoundException(tableName); } - return tableState; + return currentState.getState(); } - TableDescriptors getTableDescriptors() { - return descriptors; + protected void udpateMetaState(TableName tableName, TableState.State newState) + throws IOException { + MetaTableAccessor.updateTableState(master.getConnection(), tableName, newState); } - /** - * Write descriptor in place, update cache of states. - * Write lock should be hold by caller. - * - * @param descriptor what to write - */ - private void writeDescriptor(TableDescriptor descriptor) throws IOException { - TableName tableName = descriptor.getHTableDescriptor().getTableName(); - TableState.State state = descriptor.getTableState(); - descriptors.add(descriptor); - LOG.debug("Table " + tableName + " written descriptor for state " + state); - tableStates.put(tableName, state); - LOG.debug("Table " + tableName + " updated state to " + state); + @Nullable + protected TableState readMetaState(TableName tableName) throws IOException { + if (tableName.equals(TableName.META_TABLE_NAME)) + return new TableState(tableName, TableState.State.ENABLED); + return MetaTableAccessor.getTableState(master.getConnection(), tableName); } - /** - * Read current descriptor for table, update cache of states. - * - * @param table descriptor to read - * @return descriptor - * @throws IOException - */ - private TableDescriptor readDescriptor(TableName tableName) throws IOException { - TableDescriptor descriptor = descriptors.getDescriptor(tableName); - if (descriptor == null) - tableStates.remove(tableName); - else - tableStates.put(tableName, descriptor.getTableState()); - return descriptor; + @SuppressWarnings("deprecation") + public void start() throws IOException { + TableDescriptors tableDescriptors = master.getTableDescriptors(); + Connection connection = master.getConnection(); + fixTableStates(tableDescriptors, connection); + } + + public static void fixTableStates(TableDescriptors tableDescriptors, Connection connection) + throws IOException { + final Map<String, TableDescriptor> allDescriptors = + tableDescriptors.getAllDescriptors(); + final Map<String, TableState> states = new HashMap<>(); + MetaTableAccessor.fullScanTables(connection, new MetaTableAccessor.Visitor() { + @Override + public boolean visit(Result r) throws IOException { + TableState state = MetaTableAccessor.getTableState(r); + if (state != null) + states.put(state.getTableName().getNameAsString(), state); + return true; + } + }); + for (Map.Entry<String, TableDescriptor> entry : allDescriptors.entrySet()) { + String table = entry.getKey(); + if (table.equals(TableName.META_TABLE_NAME.getNameAsString())) + continue; + if (!states.containsKey(table)) { + LOG.warn("Found table without state " + table); + TableDescriptor td = entry.getValue(); + TableState.State tds = td.getTableState(); + if (tds != null) { + LOG.warn("Found table with state in descriptor, using that state"); + MetaTableAccessor.updateTableState(connection, TableName.valueOf(table), tds); + LOG.warn("Updating table descriptor"); + td.setTableState(null); + tableDescriptors.add(td); + } else { + LOG.warn("Found table with no state in descriptor, assuming ENABLED"); + MetaTableAccessor.updateTableState(connection, TableName.valueOf(table), + TableState.State.ENABLED); + } + } + } } } http://git-wip-us.apache.org/repos/asf/hbase/blob/fa852c4c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java index 2007ed4..b60733e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java @@ -70,6 +70,7 @@ public class CreateTableHandler extends EventHandler { private final AssignmentManager assignmentManager; private final TableLockManager tableLockManager; private final HRegionInfo [] newRegions; + private final MasterServices masterServices; private final TableLock tableLock; private User activeUser; @@ -82,6 +83,7 @@ public class CreateTableHandler extends EventHandler { this.hTableDescriptor = hTableDescriptor; this.conf = conf; this.newRegions = newRegions; + this.masterServices = masterServices; this.assignmentManager = masterServices.getAssignmentManager(); this.tableLockManager = masterServices.getTableLockManager(); @@ -209,10 +211,11 @@ public class CreateTableHandler extends EventHandler { // 1. Create Table Descriptor // using a copy of descriptor, table will be created enabling first TableDescriptor underConstruction = new TableDescriptor( - this.hTableDescriptor, TableState.State.ENABLING); + this.hTableDescriptor); Path tempTableDir = FSUtils.getTableDir(tempdir, tableName); - new FSTableDescriptors(this.conf).createTableDescriptorForTableDirectory( - tempTableDir, underConstruction, false); + ((FSTableDescriptors)(masterServices.getTableDescriptors())) + .createTableDescriptorForTableDirectory( + tempTableDir, underConstruction, false); Path tableDir = FSUtils.getTableDir(fileSystemManager.getRootDir(), tableName); // 2. Create Regions @@ -223,6 +226,12 @@ public class CreateTableHandler extends EventHandler { " to hbase root=" + tableDir); } + // populate descriptors cache to be visible in getAll + masterServices.getTableDescriptors().get(tableName); + + MetaTableAccessor.updateTableState(this.server.getConnection(), hTableDescriptor.getTableName(), + TableState.State.ENABLING); + if (regionInfos != null && regionInfos.size() > 0) { // 4. Add regions to META addRegionsToMeta(regionInfos, hTableDescriptor.getRegionReplication()); http://git-wip-us.apache.org/repos/asf/hbase/blob/fa852c4c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TruncateTableHandler.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TruncateTableHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TruncateTableHandler.java index ee40153..15a5b8f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TruncateTableHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TruncateTableHandler.java @@ -95,10 +95,10 @@ public class TruncateTableHandler extends DeleteTableHandler { AssignmentManager assignmentManager = this.masterServices.getAssignmentManager(); // 1. Create Table Descriptor - TableDescriptor underConstruction = new TableDescriptor( - this.hTableDescriptor, TableState.State.ENABLING); + TableDescriptor underConstruction = new TableDescriptor(this.hTableDescriptor); Path tempTableDir = FSUtils.getTableDir(tempdir, this.tableName); - new FSTableDescriptors(server.getConfiguration()) + + ((FSTableDescriptors)(masterServices.getTableDescriptors())) .createTableDescriptorForTableDirectory(tempTableDir, underConstruction, false); Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), this.tableName); @@ -123,6 +123,11 @@ public class TruncateTableHandler extends DeleteTableHandler { " to hbase root=" + tableDir); } + // populate descriptors cache to be visible in getAll + masterServices.getTableDescriptors().get(tableName); + + assignmentManager.getTableStateManager().setTableState(tableName, + TableState.State.ENABLING); // 4. Add regions to META MetaTableAccessor.addRegionsToMeta(masterServices.getConnection(), regionInfos, hTableDescriptor.getRegionReplication()); http://git-wip-us.apache.org/repos/asf/hbase/blob/fa852c4c/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 90b29ef..c170a65 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -528,8 +528,7 @@ public class HRegionServer extends HasThread implements boolean useHBaseChecksum = conf.getBoolean(HConstants.HBASE_CHECKSUM_VERIFICATION, true); this.fs = new HFileSystem(this.conf, useHBaseChecksum); this.rootDir = FSUtils.getRootDir(this.conf); - this.tableDescriptors = new FSTableDescriptors(this.conf, - this.fs, this.rootDir, !canUpdateTableDescriptor(), false); + this.tableDescriptors = getFsTableDescriptors(); service = new ExecutorService(getServerName().toShortString()); spanReceiverHost = SpanReceiverHost.getInstance(getConfiguration()); @@ -561,6 +560,11 @@ public class HRegionServer extends HasThread implements this.choreService = new ChoreService(getServerName().toString()); } + protected TableDescriptors getFsTableDescriptors() throws IOException { + return new FSTableDescriptors(this.conf, + this.fs, this.rootDir, !canUpdateTableDescriptor(), false); + } + protected void login(UserProvider user, String host) throws IOException { user.login("hbase.regionserver.keytab.file", "hbase.regionserver.kerberos.principal", host); http://git-wip-us.apache.org/repos/asf/hbase/blob/fa852c4c/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java index a3cfa04..330ead4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java @@ -39,7 +39,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableDescriptor; -import org.apache.hadoop.hbase.client.TableState; import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotDataManifest; @@ -357,7 +356,7 @@ public class SnapshotManifest { // write a copy of descriptor to the snapshot directory new FSTableDescriptors(conf, fs, rootDir) .createTableDescriptorForTableDirectory(workingDir, new TableDescriptor( - htd, TableState.State.ENABLED), false); + htd), false); } else { LOG.debug("Convert to Single Snapshot Manifest"); convertToV2SingleManifest(); http://git-wip-us.apache.org/repos/asf/hbase/blob/fa852c4c/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java index 7a6811c..cce37d7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java @@ -33,7 +33,6 @@ import com.google.common.primitives.Ints; import org.apache.commons.lang.NotImplementedException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -47,7 +46,7 @@ import org.apache.hadoop.hbase.TableDescriptor; import org.apache.hadoop.hbase.TableDescriptors; import org.apache.hadoop.hbase.TableInfoMissingException; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.TableState; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; @@ -154,7 +153,7 @@ public class FSTableDescriptors implements TableDescriptors { invocations++; if (TableName.META_TABLE_NAME.equals(tablename)) { cachehits++; - return new TableDescriptor(metaTableDescritor, TableState.State.ENABLED); + return new TableDescriptor(metaTableDescritor); } // hbase:meta is already handled. If some one tries to get the descriptor for // .logs, .oldlogs or .corrupt throw an exception. @@ -218,7 +217,7 @@ public class FSTableDescriptors implements TableDescriptors { } // add hbase:meta to the response tds.put(this.metaTableDescritor.getNameAsString(), - new TableDescriptor(metaTableDescritor, TableState.State.ENABLED)); + new TableDescriptor(metaTableDescritor)); } else { LOG.debug("Fetching table descriptors from the filesystem."); boolean allvisited = true; @@ -592,7 +591,7 @@ public class FSTableDescriptors implements TableDescriptors { HTableDescriptor htd = HTableDescriptor.parseFrom(content); LOG.warn("Found old table descriptor, converting to new format for table " + htd.getTableName() + "; NOTE table will be in ENABLED state!"); - td = new TableDescriptor(htd, TableState.State.ENABLED); + td = new TableDescriptor(htd); if (rewritePb) rewriteTableDescriptor(fs, status, td); } catch (DeserializationException e1) { throw new IOException("content=" + Bytes.toShort(content), e); http://git-wip-us.apache.org/repos/asf/hbase/blob/fa852c4c/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java index 8e1d848..a8b60cd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java @@ -53,12 +53,16 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; +import com.google.common.collect.TreeMultimap; +import com.google.protobuf.ServiceException; import org.apache.commons.lang.RandomStringUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FSDataOutputStream; @@ -85,15 +89,15 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ClusterConnection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HConnectable; import org.apache.hadoop.hbase.client.HConnection; -import org.apache.hadoop.hbase.client.HConnectionManager; import org.apache.hadoop.hbase.client.MetaScanner; import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor; import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase; @@ -137,13 +141,6 @@ import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; import org.apache.zookeeper.KeeperException; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Multimap; -import com.google.common.collect.TreeMultimap; -import com.google.protobuf.ServiceException; - /** * HBaseFsck (hbck) is a tool for checking and repairing region consistency and * table integrity problems in a corrupted HBase. @@ -245,7 +242,8 @@ public class HBaseFsck extends Configured implements Closeable { // hbase:meta are always checked private Set<TableName> tablesIncluded = new HashSet<TableName>(); private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge - private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE; // maximum number of overlapping regions to sideline + // maximum number of overlapping regions to sideline + private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE; private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions private Path sidelineDir = null; @@ -267,8 +265,6 @@ public class HBaseFsck extends Configured implements Closeable { * to detect and correct consistency (hdfs/meta/deployment) problems. */ private TreeMap<String, HbckInfo> regionInfoMap = new TreeMap<String, HbckInfo>(); - private TreeSet<TableName> disabledTables = - new TreeSet<TableName>(); // Empty regioninfo qualifiers in hbase:meta private Set<Result> emptyRegionInfoQualifiers = new HashSet<Result>(); @@ -292,6 +288,8 @@ public class HBaseFsck extends Configured implements Closeable { private Map<TableName, Set<String>> orphanTableDirs = new HashMap<TableName, Set<String>>(); + private Map<TableName, TableState> tableStates = + new HashMap<TableName, TableState>(); /** * Constructor @@ -493,7 +491,7 @@ public class HBaseFsck extends Configured implements Closeable { fixes = 0; regionInfoMap.clear(); emptyRegionInfoQualifiers.clear(); - disabledTables.clear(); + tableStates.clear(); errors.clear(); tablesInfo.clear(); orphanHdfsDirs.clear(); @@ -577,15 +575,15 @@ public class HBaseFsck extends Configured implements Closeable { reportTablesInFlux(); } + // Get disabled tables states + loadTableStates(); + // load regiondirs and regioninfos from HDFS if (shouldCheckHdfs()) { loadHdfsRegionDirs(); loadHdfsRegionInfos(); } - // Get disabled tables from ZooKeeper - loadDisabledTables(); - // fix the orphan tables fixOrphanTables(); @@ -1140,7 +1138,7 @@ public class HBaseFsck extends Configured implements Closeable { for (String columnfamimly : columns) { htd.addFamily(new HColumnDescriptor(columnfamimly)); } - fstd.createTableDescriptor(new TableDescriptor(htd, TableState.State.ENABLED), true); + fstd.createTableDescriptor(new TableDescriptor(htd), true); return true; } @@ -1188,7 +1186,7 @@ public class HBaseFsck extends Configured implements Closeable { if (tableName.equals(htds[j].getTableName())) { HTableDescriptor htd = htds[j]; LOG.info("fixing orphan table: " + tableName + " from cache"); - fstd.createTableDescriptor(new TableDescriptor(htd, TableState.State.ENABLED), true); + fstd.createTableDescriptor(new TableDescriptor(htd), true); j++; iter.remove(); } @@ -1265,6 +1263,8 @@ public class HBaseFsck extends Configured implements Closeable { } TableInfo ti = e.getValue(); + puts.add(MetaTableAccessor + .makePutFromTableState(new TableState(ti.tableName, TableState.State.ENABLED))); for (Entry<byte[], Collection<HbckInfo>> spl : ti.sc.getStarts().asMap() .entrySet()) { Collection<HbckInfo> his = spl.getValue(); @@ -1524,28 +1524,19 @@ public class HBaseFsck extends Configured implements Closeable { * @throws ZooKeeperConnectionException * @throws IOException */ - private void loadDisabledTables() + private void loadTableStates() throws IOException { - HConnectionManager.execute(new HConnectable<Void>(getConf()) { - @Override - public Void connect(HConnection connection) throws IOException { - TableName[] tables = connection.listTableNames(); - for (TableName table : tables) { - if (connection.getTableState(table) - .inStates(TableState.State.DISABLED, TableState.State.DISABLING)) { - disabledTables.add(table); - } - } - return null; - } - }); + tableStates = MetaTableAccessor.getTableStates(connection); } /** * Check if the specified region's table is disabled. + * @param tableName table to check status of */ - private boolean isTableDisabled(HRegionInfo regionInfo) { - return disabledTables.contains(regionInfo.getTable()); + private boolean isTableDisabled(TableName tableName) { + return tableStates.containsKey(tableName) + && tableStates.get(tableName) + .inStates(TableState.State.DISABLED, TableState.State.DISABLING); } /** @@ -1615,15 +1606,24 @@ public class HBaseFsck extends Configured implements Closeable { HConstants.EMPTY_START_ROW, false, false); if (rl == null) { errors.reportError(ERROR_CODE.NULL_META_REGION, - "META region or some of its attributes are null."); + "META region was not found in Zookeeper"); return false; } for (HRegionLocation metaLocation : rl.getRegionLocations()) { // Check if Meta region is valid and existing - if (metaLocation == null || metaLocation.getRegionInfo() == null || - metaLocation.getHostname() == null) { + if (metaLocation == null ) { + errors.reportError(ERROR_CODE.NULL_META_REGION, + "META region location is null"); + return false; + } + if (metaLocation.getRegionInfo() == null) { + errors.reportError(ERROR_CODE.NULL_META_REGION, + "META location regionInfo is null"); + return false; + } + if (metaLocation.getHostname() == null) { errors.reportError(ERROR_CODE.NULL_META_REGION, - "META region or some of its attributes are null."); + "META location hostName is null"); return false; } ServerName sn = metaLocation.getServerName(); @@ -1718,6 +1718,55 @@ public class HBaseFsck extends Configured implements Closeable { } } setCheckHdfs(prevHdfsCheck); + + if (shouldCheckHdfs()) { + checkAndFixTableStates(); + } + } + + /** + * Check and fix table states, assumes full info available: + * - tableInfos + * - empty tables loaded + */ + private void checkAndFixTableStates() throws IOException { + // first check dangling states + for (Entry<TableName, TableState> entry : tableStates.entrySet()) { + TableName tableName = entry.getKey(); + TableState tableState = entry.getValue(); + TableInfo tableInfo = tablesInfo.get(tableName); + if (isTableIncluded(tableName) + && !tableName.isSystemTable() + && tableInfo == null) { + if (fixMeta) { + MetaTableAccessor.deleteTableState(connection, tableName); + TableState state = MetaTableAccessor.getTableState(connection, tableName); + if (state != null) { + errors.reportError(ERROR_CODE.ORPHAN_TABLE_STATE, + tableName + " unable to delete dangling table state " + tableState); + } + } else { + errors.reportError(ERROR_CODE.ORPHAN_TABLE_STATE, + tableName + " has dangling table state " + tableState); + } + } + } + // check that all tables have states + for (TableName tableName : tablesInfo.keySet()) { + if (isTableIncluded(tableName) && !tableStates.containsKey(tableName)) { + if (fixMeta) { + MetaTableAccessor.updateTableState(connection, tableName, TableState.State.ENABLED); + TableState newState = MetaTableAccessor.getTableState(connection, tableName); + if (newState == null) { + errors.reportError(ERROR_CODE.NO_TABLE_STATE, + "Unable to change state for table " + tableName + " in meta "); + } + } else { + errors.reportError(ERROR_CODE.NO_TABLE_STATE, + tableName + " has no state in meta "); + } + } + } } private void preCheckPermission() throws IOException, AccessDeniedException { @@ -1961,8 +2010,8 @@ public class HBaseFsck extends Configured implements Closeable { hasMetaAssignment && isDeployed && !isMultiplyDeployed && hbi.metaEntry.regionServer.equals(hbi.deployedOn.get(0)); boolean splitParent = - (hbi.metaEntry == null)? false: hbi.metaEntry.isSplit() && hbi.metaEntry.isOffline(); - boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.metaEntry); + inMeta && hbi.metaEntry.isSplit() && hbi.metaEntry.isOffline(); + boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.metaEntry.getTable()); boolean recentlyModified = inHdfs && hbi.getModTime() + timelag > System.currentTimeMillis(); @@ -2744,7 +2793,7 @@ public class HBaseFsck extends Configured implements Closeable { // When table is disabled no need to check for the region chain. Some of the regions // accidently if deployed, this below code might report some issues like missing start // or end regions or region hole in chain and may try to fix which is unwanted. - if (disabledTables.contains(this.tableName)) { + if (isTableDisabled(this.tableName)) { return true; } int originalErrorsCount = errors.getErrorList().size(); @@ -3534,12 +3583,14 @@ public class HBaseFsck extends Configured implements Closeable { public interface ErrorReporter { enum ERROR_CODE { UNKNOWN, NO_META_REGION, NULL_META_REGION, NO_VERSION_FILE, NOT_IN_META_HDFS, NOT_IN_META, - NOT_IN_META_OR_DEPLOYED, NOT_IN_HDFS_OR_DEPLOYED, NOT_IN_HDFS, SERVER_DOES_NOT_MATCH_META, NOT_DEPLOYED, + NOT_IN_META_OR_DEPLOYED, NOT_IN_HDFS_OR_DEPLOYED, NOT_IN_HDFS, SERVER_DOES_NOT_MATCH_META, + NOT_DEPLOYED, MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE, FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS, HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION, ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE, - WRONG_USAGE, EMPTY_META_CELL, EXPIRED_TABLE_LOCK, BOUNDARIES_ERROR + WRONG_USAGE, EMPTY_META_CELL, EXPIRED_TABLE_LOCK, BOUNDARIES_ERROR, ORPHAN_TABLE_STATE, + NO_TABLE_STATE } void clear(); void report(String message); http://git-wip-us.apache.org/repos/asf/hbase/blob/fa852c4c/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 37114b9..1a377fc 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -18,6 +18,11 @@ package org.apache.hadoop.hbase; import javax.annotation.Nullable; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + import java.io.File; import java.io.IOException; import java.io.OutputStream; @@ -122,10 +127,6 @@ import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.ZooKeeper; import org.apache.zookeeper.ZooKeeper.States; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - /** * Facility for testing HBase. Replacement for * old HBaseTestCase and HBaseClusterTestCase functionality. @@ -2292,6 +2293,8 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { Table meta = (HTable) getConnection().getTable(TableName.META_TABLE_NAME); Arrays.sort(startKeys, Bytes.BYTES_COMPARATOR); List<HRegionInfo> newRegions = new ArrayList<HRegionInfo>(startKeys.length); + MetaTableAccessor + .updateTableState(getConnection(), htd.getTableName(), TableState.State.ENABLED); // add custom ones for (int i = 0; i < startKeys.length; i++) { int j = (i + 1) % startKeys.length; @@ -2953,17 +2956,36 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { public String explainTableState(final TableName table, TableState.State state) throws IOException { - TableState.State tableState = - getHBaseCluster().getMaster().getTableStateManager().getTableState(table); + TableState tableState = MetaTableAccessor.getTableState(connection, table); if (tableState == null) { - return "TableState: No state for table " + table; - } else if (!tableState.equals(state)) { - return "TableState: Not " + state + " state, but " + tableState; + return "TableState in META: No table state in META for table " + table + + " last state in meta (including deleted is " + findLastTableState(table) + ")"; + } else if (!tableState.inStates(state)) { + return "TableState in META: Not " + state + " state, but " + tableState; } else { - return "TableState: OK"; + return "TableState in META: OK"; } } + @Nullable + public TableState findLastTableState(final TableName table) throws IOException { + final AtomicReference<TableState> lastTableState = new AtomicReference<>(null); + MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() { + @Override + public boolean visit(Result r) throws IOException { + if (!Arrays.equals(r.getRow(), table.getName())) + return false; + TableState state = MetaTableAccessor.getTableState(r); + if (state != null) + lastTableState.set(state); + return true; + } + }; + MetaTableAccessor + .fullScan(connection, visitor, table.getName(), MetaTableAccessor.QueryType.TABLE, true); + return lastTableState.get(); + } + /** * Waits for a table to be 'enabled'. Enabled means that table is set as 'enabled' and the * regions have been all assigned. Will timeout after default period (30 seconds) http://git-wip-us.apache.org/repos/asf/hbase/blob/fa852c4c/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java index e637976..eefb974 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java @@ -228,7 +228,12 @@ public class TestMetaTableAccessor { admin.deleteTable(name); assertFalse(MetaTableAccessor.tableExists(connection, name)); assertTrue(MetaTableAccessor.tableExists(connection, - TableName.META_TABLE_NAME)); + TableName.META_TABLE_NAME)); + UTIL.createTable(name, HConstants.CATALOG_FAMILY); + assertTrue(MetaTableAccessor.tableExists(connection, name)); + admin.disableTable(name); + admin.deleteTable(name); + assertFalse(MetaTableAccessor.tableExists(connection, name)); } @Test public void testGetRegion() throws IOException, InterruptedException { http://git-wip-us.apache.org/repos/asf/hbase/blob/fa852c4c/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java index 85fbbc6..fd1eff7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.client; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -248,7 +249,8 @@ public class TestAdmin1 { this.admin.disableTable(ht.getName()); assertTrue("Table must be disabled.", TEST_UTIL.getHBaseCluster() .getMaster().getAssignmentManager().getTableStateManager().isTableState( - ht.getName(), TableState.State.DISABLED)); + ht.getName(), TableState.State.DISABLED)); + assertEquals(TableState.State.DISABLED, getStateFromMeta(table)); // Test that table is disabled get = new Get(row); @@ -275,7 +277,8 @@ public class TestAdmin1 { this.admin.enableTable(table); assertTrue("Table must be enabled.", TEST_UTIL.getHBaseCluster() .getMaster().getAssignmentManager().getTableStateManager().isTableState( - ht.getName(), TableState.State.ENABLED)); + ht.getName(), TableState.State.ENABLED)); + assertEquals(TableState.State.ENABLED, getStateFromMeta(table)); // Test that table is enabled try { @@ -287,6 +290,13 @@ public class TestAdmin1 { ht.close(); } + private TableState.State getStateFromMeta(TableName table) throws IOException { + TableState state = + MetaTableAccessor.getTableState(TEST_UTIL.getConnection(), table); + assertNotNull(state); + return state.getState(); + } + @Test (timeout=300000) public void testDisableAndEnableTables() throws IOException { final byte [] row = Bytes.toBytes("row"); @@ -318,6 +328,10 @@ public class TestAdmin1 { ok = true; } + assertEquals(TableState.State.DISABLED, getStateFromMeta(table1)); + assertEquals(TableState.State.DISABLED, getStateFromMeta(table2)); + + assertTrue(ok); this.admin.enableTables("testDisableAndEnableTable.*"); @@ -336,18 +350,23 @@ public class TestAdmin1 { ht1.close(); ht2.close(); + + assertEquals(TableState.State.ENABLED, getStateFromMeta(table1)); + assertEquals(TableState.State.ENABLED, getStateFromMeta(table2)); } @Test (timeout=300000) public void testCreateTable() throws IOException { HTableDescriptor [] tables = admin.listTables(); int numTables = tables.length; - TEST_UTIL.createTable(TableName.valueOf("testCreateTable"), HConstants.CATALOG_FAMILY).close(); + TableName tableName = TableName.valueOf("testCreateTable"); + TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close(); tables = this.admin.listTables(); assertEquals(numTables + 1, tables.length); assertTrue("Table must be enabled.", TEST_UTIL.getHBaseCluster() .getMaster().getAssignmentManager().getTableStateManager().isTableState( - TableName.valueOf("testCreateTable"), TableState.State.ENABLED)); + tableName, TableState.State.ENABLED)); + assertEquals(TableState.State.ENABLED, getStateFromMeta(tableName)); } @Test (timeout=300000) @@ -405,6 +424,7 @@ public class TestAdmin1 { Table table = TEST_UTIL.getConnection().getTable(htd.getTableName()); HTableDescriptor confirmedHtd = table.getTableDescriptor(); assertEquals(htd.compareTo(confirmedHtd), 0); + MetaTableAccessor.fullScanMetaAndPrint(TEST_UTIL.getConnection()); table.close(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/fa852c4c/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java index 83ff822..8e60353 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java @@ -18,10 +18,8 @@ */ package org.apache.hadoop.hbase.client; -import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertErrors; -import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck; -import static org.junit.Assert.*; - +import javax.annotation.Nullable; +import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.List; @@ -39,21 +37,28 @@ import org.apache.hadoop.hbase.RegionLocations; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.regionserver.StorefileRefresherChore; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.HBaseFsck; -import org.apache.hadoop.hbase.util.HBaseFsckRepair; import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE; +import org.apache.hadoop.hbase.util.HBaseFsckRepair; import org.apache.hadoop.hbase.util.hbck.HbckTestingUtil; import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker; import org.apache.hadoop.hbase.zookeeper.ZKUtil; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; +import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertErrors; +import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + /** * Tests the scenarios where replicas are enabled for the meta table */ @@ -224,7 +229,8 @@ public class TestMetaWithReplicas { stopMasterAndValidateReplicaCount(2, 3); } - private void stopMasterAndValidateReplicaCount(int originalReplicaCount, int newReplicaCount) + private void stopMasterAndValidateReplicaCount(final int originalReplicaCount, + final int newReplicaCount) throws Exception { ServerName sn = TEST_UTIL.getHBaseClusterInterface().getClusterStatus().getMaster(); TEST_UTIL.getHBaseClusterInterface().stopMaster(sn); @@ -235,16 +241,7 @@ public class TestMetaWithReplicas { newReplicaCount); TEST_UTIL.getHBaseClusterInterface().startMaster(sn.getHostname(), 0); TEST_UTIL.getHBaseClusterInterface().waitForActiveAndReadyMaster(); - int count = 0; - do { - metaZnodes = TEST_UTIL.getZooKeeperWatcher().getMetaReplicaNodes(); - Thread.sleep(10); - count++; - // wait for the count to be different from the originalReplicaCount. When the - // replica count is reduced, that will happen when the master unassigns excess - // replica, and deletes the excess znodes - } while (metaZnodes.size() == originalReplicaCount && count < 1000); - assert(metaZnodes.size() == newReplicaCount); + TEST_UTIL.waitFor(10000, predicateMetaHasReplicas(newReplicaCount)); // also check if hbck returns without errors TEST_UTIL.getConfiguration().setInt(HConstants.META_REPLICAS_NUM, newReplicaCount); @@ -252,6 +249,46 @@ public class TestMetaWithReplicas { HbckTestingUtil.assertNoErrors(hbck); } + private Waiter.ExplainingPredicate<Exception> predicateMetaHasReplicas( + final int newReplicaCount) { + return new Waiter.ExplainingPredicate<Exception>() { + @Override + public String explainFailure() throws Exception { + return checkMetaLocationAndExplain(newReplicaCount); + } + + @Override + public boolean evaluate() throws Exception { + return checkMetaLocationAndExplain(newReplicaCount) == null; + } + }; + } + + @Nullable + private String checkMetaLocationAndExplain(int originalReplicaCount) + throws KeeperException, IOException { + List<String> metaZnodes = TEST_UTIL.getZooKeeperWatcher().getMetaReplicaNodes(); + if (metaZnodes.size() == originalReplicaCount) { + RegionLocations rl = ((ClusterConnection) TEST_UTIL.getConnection()) + .locateRegion(TableName.META_TABLE_NAME, + HConstants.EMPTY_START_ROW, false, false); + for (HRegionLocation location : rl.getRegionLocations()) { + if (location == null) { + return "Null location found in " + rl.toString(); + } + if (location.getRegionInfo() == null) { + return "Null regionInfo for location " + location; + } + if (location.getHostname() == null) { + return "Null hostName for location " + location; + } + } + return null; // OK + } + return "Replica count is not as expected " + originalReplicaCount + " <> " + metaZnodes.size() + + "(" + metaZnodes.toString() + ")"; + } + @Test public void testHBaseFsckWithMetaReplicas() throws Exception { HBaseFsck hbck = HbckTestingUtil.doFsck(TEST_UTIL.getConfiguration(), false); http://git-wip-us.apache.org/repos/asf/hbase/blob/fa852c4c/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestReplicaWithCluster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestReplicaWithCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestReplicaWithCluster.java index 6d98c52..b0bd6f6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestReplicaWithCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestReplicaWithCluster.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.regionserver.TestHRegionServerBulkLoad; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSTableDescriptors; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; import org.junit.AfterClass; @@ -130,7 +131,8 @@ public class TestReplicaWithCluster { @AfterClass public static void afterClass() throws Exception { - HTU2.shutdownMiniCluster(); + if (HTU2 != null) + HTU2.shutdownMiniCluster(); HTU.shutdownMiniCluster(); } @@ -213,7 +215,6 @@ public class TestReplicaWithCluster { SlowMeCopro.sleepTime.set(0); } - HTU.getHBaseCluster().stopMaster(0); Admin admin = HTU.getHBaseAdmin(); nHdt =admin.getTableDescriptor(hdt.getTableName()); Assert.assertEquals("fams=" + Arrays.toString(nHdt.getColumnFamilies()), @@ -221,7 +222,6 @@ public class TestReplicaWithCluster { admin.disableTable(hdt.getTableName()); admin.deleteTable(hdt.getTableName()); - HTU.getHBaseCluster().startMaster(); admin.close(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/fa852c4c/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java index f1c080d..eb72220 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java @@ -643,7 +643,8 @@ public class TestAssignmentManagerOnCluster { if (hri != null && serverName != null) { am.regionOnline(hri, serverName); } - am.getTableStateManager().setTableState(table, TableState.State.DISABLED); + am.getTableStateManager().setTableState(table, TableState.State.ENABLED); + TEST_UTIL.getHBaseAdmin().disableTable(table); TEST_UTIL.deleteTable(table); } } @@ -1164,7 +1165,7 @@ public class TestAssignmentManagerOnCluster { tableNameList.add(TableName.valueOf(name + "_" + i)); } } - List<Result> metaRows = MetaTableAccessor.fullScanOfMeta(admin.getConnection()); + List<Result> metaRows = MetaTableAccessor.fullScanRegions(admin.getConnection()); int count = 0; // Check all 100 rows are in meta for (Result result : metaRows) { http://git-wip-us.apache.org/repos/asf/hbase/blob/fa852c4c/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java index e09583a..8ed49ff 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java @@ -1043,8 +1043,7 @@ public class TestCatalogJanitor { } private TableDescriptor createTableDescriptor() { - TableDescriptor htd = new TableDescriptor(createHTableDescriptor(), TableState.State.ENABLED); - return htd; + return new TableDescriptor(createHTableDescriptor()); } private MultiResponse buildMultiResponse(MultiRequest req) { http://git-wip-us.apache.org/repos/asf/hbase/blob/fa852c4c/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java index 6307c4c..ca9bc9c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java @@ -47,8 +47,6 @@ import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Table; @@ -134,7 +132,7 @@ public class TestMasterOperationsForRegionReplicas { } } - List<Result> metaRows = MetaTableAccessor.fullScanOfMeta(ADMIN.getConnection()); + List<Result> metaRows = MetaTableAccessor.fullScanRegions(ADMIN.getConnection()); int numRows = 0; for (Result result : metaRows) { RegionLocations locations = MetaTableAccessor.getRegionLocations(result); @@ -297,7 +295,7 @@ public class TestMasterOperationsForRegionReplicas { return true; } }; - MetaTableAccessor.fullScan(connection, visitor); + MetaTableAccessor.fullScanRegions(connection, visitor); assert(count.get() == numRegions); } http://git-wip-us.apache.org/repos/asf/hbase/blob/fa852c4c/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionStates.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionStates.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionStates.java index ce61e40..99e1709 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionStates.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionStates.java @@ -56,7 +56,7 @@ public class TestRegionStates { @Test (timeout=10000) public void testCanMakeProgressThoughMetaIsDown() throws IOException, InterruptedException, BrokenBarrierException { - Server server = mock(Server.class); + MasterServices server = mock(MasterServices.class); when(server.getServerName()).thenReturn(ServerName.valueOf("master,1,1")); Connection connection = mock(ClusterConnection.class); // Set up a table that gets 'stuck' when we try to fetch a row from the meta table. @@ -101,7 +101,7 @@ public class TestRegionStates { @Test public void testWeDontReturnDrainingServersForOurBalancePlans() throws Exception { - Server server = mock(Server.class); + MasterServices server = mock(MasterServices.class); when(server.getServerName()).thenReturn(ServerName.valueOf("master,1,1")); Configuration configuration = mock(Configuration.class); when(server.getConfiguration()).thenReturn(configuration); http://git-wip-us.apache.org/repos/asf/hbase/blob/fa852c4c/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java index 8296e81..51436b4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java @@ -2520,6 +2520,7 @@ public class TestAccessController extends SecureTestUtil { assertTrue(existingPerms.size() > 1); TEST_UTIL.getHBaseAdmin().disableTable(TEST_TABLE.getTableName()); TEST_UTIL.truncateTable(TEST_TABLE.getTableName()); + TEST_UTIL.waitTableAvailable(TEST_TABLE.getTableName()); List<UserPermission> perms = AccessControlClient.getUserPermissions(conf, TEST_TABLE.getTableName().getNameAsString()); assertTrue(perms != null); http://git-wip-us.apache.org/repos/asf/hbase/blob/fa852c4c/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java index 9a7db90..7600388 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java @@ -212,6 +212,7 @@ public class TestCoprocessorScanPolicy { // should be gone now assertEquals(0, r.size()); t.close(); + EnvironmentEdgeManager.reset(); } public static class ScanObserver extends BaseRegionObserver { http://git-wip-us.apache.org/repos/asf/hbase/blob/fa852c4c/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java index a99daf2..c09982e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java @@ -76,7 +76,7 @@ public class TestFSTableDescriptors { public void testCreateAndUpdate() throws IOException { Path testdir = UTIL.getDataTestDir("testCreateAndUpdate"); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testCreate")); - TableDescriptor td = new TableDescriptor(htd, TableState.State.ENABLED); + TableDescriptor td = new TableDescriptor(htd); FileSystem fs = FileSystem.get(UTIL.getConfiguration()); FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, testdir); assertTrue(fstd.createTableDescriptor(td)); @@ -113,7 +113,7 @@ public class TestFSTableDescriptors { assertTrue(!fs.exists(p1)); int i2 = FSTableDescriptors.getTableInfoSequenceId(p2); assertTrue(i2 == i1 + 1); - td = new TableDescriptor(htd, TableState.State.DISABLED); + td = new TableDescriptor(htd); Path p3 = fstd.updateTableDescriptor(td); // Assert we cleaned up the old file. assertTrue(!fs.exists(p2)); @@ -172,7 +172,7 @@ public class TestFSTableDescriptors { final String name = "testReadingHTDFromFS"; FileSystem fs = FileSystem.get(UTIL.getConfiguration()); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name)); - TableDescriptor td = new TableDescriptor(htd, TableState.State.ENABLED); + TableDescriptor td = new TableDescriptor(htd); Path rootdir = UTIL.getDataTestDir(name); FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir); fstd.createTableDescriptor(td); @@ -187,7 +187,7 @@ public class TestFSTableDescriptors { Path rootdir = UTIL.getDataTestDir(name); FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name)); - TableDescriptor td = new TableDescriptor(htd, TableState.State.ENABLED); + TableDescriptor td = new TableDescriptor(htd); Path descriptorFile = fstd.updateTableDescriptor(td); try (FSDataOutputStream out = fs.create(descriptorFile, true)) { out.write(htd.toByteArray()); @@ -222,8 +222,8 @@ public class TestFSTableDescriptors { final int count = 10; // Write out table infos. for (int i = 0; i < count; i++) { - TableDescriptor htd = new TableDescriptor(new HTableDescriptor(name + i), - TableState.State.ENABLED); + TableDescriptor htd = new TableDescriptor( + new HTableDescriptor(TableName.valueOf(name + i))); htds.createTableDescriptor(htd); } @@ -420,7 +420,7 @@ public class TestFSTableDescriptors { Path testdir = UTIL.getDataTestDir("testCreateTableDescriptorUpdatesIfThereExistsAlready"); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf( "testCreateTableDescriptorUpdatesIfThereExistsAlready")); - TableDescriptor td = new TableDescriptor(htd, TableState.State.ENABLED); + TableDescriptor td = new TableDescriptor(htd); FileSystem fs = FileSystem.get(UTIL.getConfiguration()); FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, testdir); assertTrue(fstd.createTableDescriptor(td)); http://git-wip-us.apache.org/repos/asf/hbase/blob/fa852c4c/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java index 33bd337..0d3a94e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java @@ -157,6 +157,7 @@ public class TestHBaseFsck { conf.setInt("hbase.hconnection.threads.max", 2 * POOL_SIZE); conf.setInt("hbase.hconnection.threads.core", POOL_SIZE); conf.setInt("hbase.hbck.close.timeout", 2 * REGION_ONLINE_TIMEOUT); + conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 2 * REGION_ONLINE_TIMEOUT); TEST_UTIL.startMiniCluster(3); tableExecutorService = new ThreadPoolExecutor(1, POOL_SIZE, 60, TimeUnit.SECONDS, @@ -1402,7 +1403,7 @@ public class TestHBaseFsck { HBaseFsck hbck = doFsck(conf, false); assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS, ERROR_CODE.NOT_IN_HDFS, ERROR_CODE.NOT_IN_HDFS, - ERROR_CODE.NOT_IN_HDFS,}); + ERROR_CODE.NOT_IN_HDFS, ERROR_CODE.ORPHAN_TABLE_STATE, }); // holes are separate from overlap groups assertEquals(0, hbck.getOverlapGroups(table).size()); @@ -1445,6 +1446,34 @@ public class TestHBaseFsck { } /** + * when the hbase.version file missing, It is fix the fault. + */ + @Test (timeout=180000) + public void testNoTableState() throws Exception { + // delete the hbase.version file + TableName table = + TableName.valueOf("testNoTableState"); + try { + setupTable(table); + // make sure data in regions, if in wal only there is no data loss + admin.flush(table); + + MetaTableAccessor.deleteTableState(TEST_UTIL.getConnection(), table); + + // test + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_TABLE_STATE }); + // fix table state missing + doFsck(conf, true); + + assertNoErrors(doFsck(conf, false)); + assertTrue(TEST_UTIL.getHBaseAdmin().isTableEnabled(table)); + } finally { + cleanupTable(table); + } + } + + /** * The region is not deployed when the table is disabled. */ @Test (timeout=180000) http://git-wip-us.apache.org/repos/asf/hbase/blob/fa852c4c/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRebuildTestCore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRebuildTestCore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRebuildTestCore.java index 349bf56..e940425 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRebuildTestCore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRebuildTestCore.java @@ -285,16 +285,9 @@ public class OfflineMetaRebuildTestCore { * @return # of entries in meta. */ protected int scanMeta() throws IOException { - int count = 0; - Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME); - ResultScanner scanner = meta.getScanner(new Scan()); - LOG.info("Table: " + meta.getName()); - for (Result res : scanner) { - LOG.info(Bytes.toString(res.getRow())); - count++; - } - meta.close(); - return count; + LOG.info("Scanning META"); + MetaTableAccessor.fullScanMetaAndPrint(TEST_UTIL.getConnection()); + return MetaTableAccessor.fullScanRegions(TEST_UTIL.getConnection()).size(); } protected HTableDescriptor[] getTables(final Configuration configuration) throws IOException { http://git-wip-us.apache.org/repos/asf/hbase/blob/fa852c4c/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java index a3d323c..fc22292 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java @@ -20,11 +20,13 @@ package org.apache.hadoop.hbase.util.hbck; import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertErrors; import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import java.util.Arrays; import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; @@ -70,13 +72,20 @@ public class TestOfflineMetaRebuildBase extends OfflineMetaRebuildTestCore { TEST_UTIL.restartHBaseCluster(3); try (Connection connection = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())) { Admin admin = connection.getAdmin(); - admin.enableTable(table); + if (admin.isTableDisabled(table)) + admin.enableTable(table); LOG.info("Waiting for no more RIT"); TEST_UTIL.waitUntilNoRegionsInTransition(60000); LOG.info("No more RIT in ZK, now doing final test verification"); // everything is good again. - assertEquals(5, scanMeta()); + assertEquals(5, scanMeta()); // including table state rows + TableName[] tableNames = TEST_UTIL.getHBaseAdmin().listTableNames(); + for (TableName tableName : tableNames) { + HTableDescriptor tableDescriptor = TEST_UTIL.getHBaseAdmin().getTableDescriptor(tableName); + assertNotNull(tableDescriptor); + assertTrue(TEST_UTIL.getHBaseAdmin().isTableEnabled(tableName)); + } HTableDescriptor[] htbls = admin.listTables(); LOG.info("Tables present after restart: " + Arrays.toString(htbls)); assertEquals(1, htbls.length);
