Author: liyin Date: Wed Apr 2 21:03:34 2014 New Revision: 1584180 URL: http://svn.apache.org/r1584180 Log: [HBASE-10865] Store per table server assignment preference
Author: elliott Summary: * Allow user to add server perference in HTableDescriptor * Honor that preference when creating the table. * Add tests for ser/deser of server lists in HTD. Test Plan: * Wrote unit tests for HTD ser/deser * Changed previous test that creates table on servers iloveunittests Reviewers: adela Reviewed By: adela CC: hbase-eng@ Differential Revision: https://phabricator.fb.com/D1246832 Task ID: 4023938 Added: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestCreateTableOnServers.java - copied, changed from r1584179, hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHackyCreateTable.java Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/HMaster.java hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHackyCreateTable.java hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/thrift/swift/TestSwiftSerDe.java Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java?rev=1584180&r1=1584179&r2=1584180&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java (original) +++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java Wed Apr 2 21:03:34 2014 @@ -35,6 +35,9 @@ import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; +import com.google.common.collect.ImmutableSet; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; @@ -50,6 +53,7 @@ import com.facebook.swift.codec.ThriftSt */ @ThriftStruct public class HTableDescriptor implements WritableComparable<HTableDescriptor> { + public static final Log LOG = LogFactory.getLog(HTableDescriptor.class); // Changes prior to version 3 were not recorded here. // Version 3 adds metadata as a map where keys and values are byte[]. @@ -92,6 +96,10 @@ public class HTableDescriptor implements public static final ImmutableBytesWritable DISABLE_WAL_KEY = new ImmutableBytesWritable(Bytes.toBytes(DISABLE_WAL)); + public static final String SERVER_SET = "SERVER_SET"; + public static final ImmutableBytesWritable SERVER_SET_KEY = + new ImmutableBytesWritable( Bytes.toBytes(SERVER_SET)); + // The below are ugly but better than creating them each time till we // replace booleans being saved as Strings with plain booleans. Need a @@ -138,6 +146,9 @@ public class HTableDescriptor implements public final Map<byte [], HColumnDescriptor> families = new TreeMap<byte [], HColumnDescriptor>(Bytes.BYTES_RAWCOMPARATOR); + // Used to store what servers this can be assigned to. + private ServerSet serverSet = null; + /** * Private constructor used internally creating table descriptors for * catalog tables: e.g. .META. and -ROOT-. @@ -821,11 +832,75 @@ public class HTableDescriptor implements } /** - * @param readOnly True if all of the columns in the table should be read + * @param disable should the wal be disabled for this table. * only. */ public void setWALDisabled(final boolean disable) { setValue(DISABLE_WAL_KEY, disable? TRUE: FALSE); } + /** + * This is the class serialized into SERVER_SET_KEY to hold the list of servers. + */ + @ThriftStruct + public static class ServerSet { + + private Set<HServerAddress> servers = null; + + @ThriftConstructor + public ServerSet(Set<HServerAddress> servers) { + this.servers = servers; + } + + @ThriftField(1) + public Set<HServerAddress> getServers() { + return servers; + } + + public void setServers(Set<HServerAddress> servers) { + this.servers = servers; + } + } + + public synchronized void setServers(Collection<HServerAddress> servers) { + if (this.serverSet == null) { + this.serverSet = new ServerSet(null); + } + + if ( servers == null) { + this.serverSet.servers = null; + return; + } + + if (servers.size() < 3 ) { + throw new IllegalArgumentException("Must provide at least three servers."); + } + + this.serverSet.setServers(ImmutableSet.copyOf(servers)); + try { + // Hex encode everything so that the HTD can still be printed everywhere. + String encodedStringSet = Bytes.writeThriftBytesAndGetString(serverSet, ServerSet.class); + this.setValue(SERVER_SET_KEY, Bytes.toBytes(encodedStringSet)); + } catch (Exception e) { + LOG.error("Error serializing server set from HTableDescriptor", e); + } + } + + public synchronized Set<HServerAddress> getServers() { + if (serverSet == null) { + byte[] serverSetBytes = getValue(SERVER_SET_KEY); + if (serverSetBytes != null) { + try { + byte[] decodedBytes = Bytes.hexToBytes(Bytes.toString(serverSetBytes)); + serverSet = Bytes.readThriftBytes(decodedBytes, ServerSet.class); + } catch (Exception e) { + LOG.error("Error de-serializing server set into HTableDescriptor", e); + } + } + if (serverSet == null) { + serverSet = new ServerSet(null); + } + } + return this.serverSet.getServers(); + } } Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java?rev=1584180&r1=1584179&r2=1584180&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java (original) +++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java Wed Apr 2 21:03:34 2014 @@ -270,20 +270,6 @@ public class HBaseAdmin { checkTableOnline(desc, splitKeys); } - /** - * Same like {@link #createTable(HTableDescriptor, byte[][])} but creates - * table on specific regionservers - * - * @throws IOException - */ - public void createTable(final HTableDescriptor desc, byte[][] splitKeys, - List<HServerAddress> servers) throws IOException { - HTableDescriptor.isLegalTableName(desc.getName()); - checkSplitKeys(splitKeys); - createTableAsyncAndPlaceOnServers(desc, splitKeys, servers); - checkTableOnline(desc, splitKeys); - } - private void checkTableOnline(final HTableDescriptor desc, byte[][] splitKeys) throws IOException, RegionOfflineException, InterruptedIOException { int numRegs = splitKeys == null ? 1 : splitKeys.length + 1; @@ -383,30 +369,6 @@ public class HBaseAdmin { } } - /** - * SAme as {@link #createTableAsync(HTableDescriptor, byte[][])} but using a - * specific set of servers to assign the regions - * - * @param desc - * @param splitKeys - * @param servers - * @throws IOException - */ - public void createTableAsyncAndPlaceOnServers(HTableDescriptor desc, - byte[][] splitKeys, List<HServerAddress> servers) throws IOException { - if (this.master == null) { - throw new MasterNotRunningException("master has been shut down"); - } - HTableDescriptor.isLegalTableName(desc.getName()); - try { - this.master.createTableAndAssignOnServers(desc, splitKeys, servers); - } catch (RemoteException e) { - throw RemoteExceptionHandler.decodeRemoteException(e); - } catch (SocketTimeoutException ste) { - LOG.warn("Creating " + desc.getNameAsString() + " took too long", ste); - } - } - /** * Deletes a table. Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java?rev=1584180&r1=1584179&r2=1584180&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java (original) +++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java Wed Apr 2 21:03:34 2014 @@ -59,17 +59,6 @@ public interface HMasterInterface extend throws IOException; /** - * Same like {@link #createTable(HTableDescriptor, byte[][])} above, but we - * specify the set of machines where we want the regions to be assigned - * - * @param desc - * @param splitKeys - * @throws IOException - */ - public void createTableAndAssignOnServers(HTableDescriptor desc, - byte[][] splitKeys, List<HServerAddress> servers) throws IOException; - - /** * Deletes a table * @param tableName table to delete * @throws IOException e Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/HMaster.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1584180&r1=1584179&r2=1584180&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original) +++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Wed Apr 2 21:03:34 2014 @@ -33,6 +33,7 @@ import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -1484,31 +1485,6 @@ public class HMaster extends HasThread i } } - @Override - public void createTableAndAssignOnServers(HTableDescriptor desc, - byte[][] splitKeys, List<HServerAddress> servers) throws IOException { - HRegionInfo[] newRegions = createRegionsForNewTable(desc, splitKeys); - try { - // We can not create a table unless meta regions have already been - // assigned and scanned. - if (!this.regionManager.areAllMetaRegionsOnline()) { - throw new NotAllMetaRegionsOnlineException(); - } - if (!this.serverManager.hasEnoughRegionServers()) { - throw new IOException("not enough servers to create table yet"); - } - createTable(newRegions, servers); - LOG.info("Succeeded in creating table " + desc.getNameAsString() - + "on the specified servers"); - } catch (TableExistsException e) { - throw e; - } catch (IOException e) { - LOG.error("Cannot create table " + desc.getNameAsString() - + " because of " + e.toString() + " on the specified servers"); - throw RemoteExceptionHandler.checkIOException(e); - } - } - private static boolean tableExists(HRegionInterface srvr, byte[] metaRegionName, String tableName) throws IOException { @@ -1534,40 +1510,23 @@ public class HMaster extends HasThread i return false; } - private synchronized void createTable(final HRegionInfo[] newRegions) - throws IOException { - String tableName = newRegions[0].getTableDesc().getNameAsString(); - AssignmentPlan assignmentPlan = null; - if (this.shouldAssignRegionsWithFavoredNodes) { - // Get the assignment domain for this table - AssignmentDomain domain = this.getAssignmentDomain(tableName); - // Get the assignment plan for the new regions - assignmentPlan = regionPlacement.getNewAssignmentPlan(newRegions, domain); - } - createTable(newRegions, assignmentPlan); - } - /** - * Create table such that we place the new regions on the specified machines + * Create table. If the HTableDescriptor has servers listed the regions will be placed + * on those servers. + * * @param newRegions - new regions from the new table - * @param servers - set of machines where we like the regions to be placed * @throws IOException */ - private synchronized void createTable(final HRegionInfo[] newRegions, - List<HServerAddress> servers) throws IOException { + private synchronized void createTable(final HRegionInfo[] newRegions) throws IOException { String tableName = newRegions[0].getTableDesc().getNameAsString(); AssignmentPlan assignmentPlan = null; if (this.shouldAssignRegionsWithFavoredNodes) { // Get the assignment domain for this table - AssignmentDomain domain = this.getAssignmentDomain(tableName, servers); + AssignmentDomain domain = this.getAssignmentDomain(newRegions[0].getTableDesc()); // Get the assignment plan for the new regions assignmentPlan = regionPlacement.getNewAssignmentPlan(newRegions, domain); } - createTable(newRegions, assignmentPlan); - } - private synchronized void createTable(final HRegionInfo[] newRegions, AssignmentPlan assignmentPlan) throws IOException { - String tableName = newRegions[0].getTableDesc().getNameAsString(); // 1. Check to see if table already exists. Get meta region where // table would sit should it exist. Open scanner on it. If a region // for the table we want to create already exists, then table already @@ -1617,30 +1576,23 @@ public class HMaster extends HasThread i * Currently the domain would be generated by shuffling all the online * region servers. * - * It would be easy to extend for the multi-tenancy in the future. - * @param tableName + * @param htd * @return the assignment domain for the table. */ - private AssignmentDomain getAssignmentDomain(String tableName) { + private AssignmentDomain getAssignmentDomain(HTableDescriptor htd) { // Get all the online region servers - List<HServerAddress> onlineRSList = - this.serverManager.getOnlineRegionServerList(); - return getAssignmentDomain(tableName, onlineRSList); - } + List<HServerAddress> servers = null; + + if (htd.getServers() == null) { + servers = this.serverManager.getOnlineRegionServerList(); + } else { + Set<HServerAddress> hServerAddresses = new HashSet<>(htd.getServers()); + hServerAddresses.retainAll(this.serverManager.getOnlineRegionServerList()); + servers = Lists.newArrayList(hServerAddresses); + } - /** - * Get the assignment domain for the table. Currently the domain would be - * generated by shuffling the passed region servers. - * - * It would be easy to extend for the multi-tenancy in the future. - * - * @param tableName - * @param servers - list of servers that we want to be included in the plan - * @return the assignment domain for the table. - */ - private AssignmentDomain getAssignmentDomain(String tableName, List<HServerAddress> servers) { // Shuffle the server list based on the tableName - Random random = new Random(tableName.hashCode()); + Random random = new Random(htd.getNameAsString().hashCode()); Collections.shuffle(servers, random); // Add the shuffled server list into the assignment domain AssignmentDomain domain = new AssignmentDomain(this.conf); Added: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java?rev=1584180&view=auto ============================================================================== --- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java (added) +++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java Wed Apr 2 21:03:34 2014 @@ -0,0 +1,120 @@ +/** + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.DataInputBuffer; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.util.HashSet; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; + +public class TestHTableDescriptor { + @Test + public void testGetNameAsString() throws Exception { + HTableDescriptor htd = new HTableDescriptor(Bytes.toBytes("Test")); + + // Make sure the to and from string works + assertEquals("Test", htd.getNameAsString()); + + // Make sure the string instance is cached. + assertSame(htd.getNameAsString(), htd.getNameAsString()); + } + + @Test + public void testWriteAndReadFieldsWithServerSet() throws Exception { + HTableDescriptor htd = new HTableDescriptor("TestTable"); + htd.addFamily(new HColumnDescriptor("d")); + htd.setMaxFileSize(Long.MAX_VALUE - 1); + htd.setWALDisabled(true); + + + Set<HServerAddress> servers = new HashSet<>(); + + servers.add(new HServerAddress(java.net.InetAddress.getLocalHost().getHostName(),0)); + htd.setServers(servers); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(baos); + htd.write(out); + + byte[] buf = baos.toByteArray(); + + DataInputBuffer in = new DataInputBuffer(); + in.reset(buf, 0, buf.length); + + HTableDescriptor n = new HTableDescriptor(); + n.readFields(in); + + assertEquals("TestTable", n.getNameAsString()); + assertEquals(servers, n.getServers()); + assertEquals(true, n.isWALDisabled()); + assertEquals(1, n.getColumnFamilies().length); + assertEquals(Long.MAX_VALUE -1, n.getMaxFileSize()); + + } + + @Test + public void testWriteAndReadFieldsWithoutServerSet() throws Exception { + HTableDescriptor htd = new HTableDescriptor("TestTable"); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(baos); + htd.write(out); + + byte[] buf = baos.toByteArray(); + + DataInputBuffer in = new DataInputBuffer(); + in.reset(buf, 0, buf.length); + + HTableDescriptor n = new HTableDescriptor(); + n.readFields(in); + + assertEquals("TestTable", n.getNameAsString()); + assertEquals(null, n.getServers()); + } + + @Test + public void testSetServers() throws Exception { + HTableDescriptor htd = new HTableDescriptor("TestTable"); + Set<HServerAddress> servers = new HashSet<>(); + + servers.add(new HServerAddress(java.net.InetAddress.getLocalHost().getHostName(),0)); + htd.setServers(servers); + + assertEquals(servers, htd.getServers()); + + htd.setServers(null); + assertEquals(null, htd.getServers()); + } + + + @Test(expected=IllegalArgumentException.class) + public void testSetServersThrowErrorNoServers() throws Exception { + HTableDescriptor htd = new HTableDescriptor("testThrowErrorNoServers"); + htd.addFamily(new HColumnDescriptor("d")); + htd.setServers(new HashSet<HServerAddress>()); + } +} Copied: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestCreateTableOnServers.java (from r1584179, hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHackyCreateTable.java) URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestCreateTableOnServers.java?p2=hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestCreateTableOnServers.java&p1=hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHackyCreateTable.java&r1=1584179&r2=1584180&rev=1584180&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHackyCreateTable.java (original) +++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestCreateTableOnServers.java Wed Apr 2 21:03:34 2014 @@ -43,18 +43,20 @@ import org.junit.Test; /** * Create table only on a set of regionservers instead of whole cluster - * JIRA: https://issues.apache.org/jira/browse/HBASE-10425 + * JIRA: * + * https://issues.apache.org/jira/browse/HBASE-10425 + * https://issues.apache.org/jira/browse/HBASE-10865 */ -public class TestHackyCreateTable { +public class TestCreateTableOnServers { final Log LOG = LogFactory.getLog(getClass()); private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private HBaseAdmin admin; - private static final int NUM_REGION_SERVER = 10; + private static final int NUM_REGION_SERVER = 7; /** * Number of regionservers that we will remove in the test from {@link #NUM_REGION_SERVER} */ - private static final int NUM_REMOVED_RS = 7; + private static final int NUM_REMOVED_RS = 4; @BeforeClass public static void setUpBeforeClass() throws Exception { TEST_UTIL.getConfiguration().set(HConstants.LOAD_BALANCER_IMPL, @@ -98,26 +100,28 @@ public class TestHackyCreateTable { }; HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); - List<HServerAddress> allServers = TEST_UTIL.getHBaseCluster().getRegionServers(); - System.out.println("initial number of servers: " + allServers.size()); + List<HServerAddress> keptServers = TEST_UTIL.getHBaseCluster().getRegionServers(); + System.out.println("initial number of servers: " + keptServers.size()); // remove some regionservers for (int i = 0; i < NUM_REMOVED_RS; i++) { - allServers.remove(0); + keptServers.remove(0); } // create a table on the remaining regionservers // refresh Admin to pickup new configuration about using favored nodes - admin.createTable(desc, splitKeys, allServers); + desc.setServers(keptServers); + admin.createTable(desc, splitKeys); HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName); Map<HRegionInfo,HServerAddress> regions = ht.getRegionsInfo(); int expectedRegions = splitKeys.length + 1; assertEquals("Tried to create " + expectedRegions + " regions " + - "but only found " + regions.size(), - expectedRegions, regions.size()); + "but only found " + regions.size(), + expectedRegions, regions.size() + ); //verify if we use only a subset of the regionservers Set<HServerAddress> serversForTable = new HashSet<HServerAddress>(); serversForTable.addAll(regions.values()); - assertEquals(NUM_REGION_SERVER - NUM_REMOVED_RS, allServers.size()); - assertEquals(allServers.size(), serversForTable.size()); + assertEquals(NUM_REGION_SERVER - NUM_REMOVED_RS, keptServers.size()); + assertEquals(keptServers.size(), serversForTable.size()); // create one more table only on allServers tableName = Bytes.toBytes("testCreateTableWithRegionsOnServers2"); @@ -127,7 +131,8 @@ public class TestHackyCreateTable { }; desc = new HTableDescriptor(tableName); desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); - admin.createTable(desc, splitKeys2, allServers); + desc.setServers(keptServers); + admin.createTable(desc, splitKeys2); ht = new HTable(TEST_UTIL.getConfiguration(), tableName); regions = ht.getRegionsInfo(); expectedRegions = splitKeys2.length + 1; @@ -137,7 +142,24 @@ public class TestHackyCreateTable { //verify if we use only a subset of the regionservers serversForTable = new HashSet<HServerAddress>(); serversForTable.addAll(regions.values()); - assertEquals(NUM_REGION_SERVER - NUM_REMOVED_RS, allServers.size()); - assertEquals(allServers.size(), serversForTable.size()); + assertEquals(NUM_REGION_SERVER - NUM_REMOVED_RS, keptServers.size()); + assertEquals(keptServers.size(), serversForTable.size()); + } + + /** + * Test to make sure that HTD without servers work as expected. And the server list is not + * stored without action from the user. + * @throws IOException + */ + @Test + public void testCreateTableNoServers() throws IOException { + String tn = "globalAssignmentDomainTable"; + HTableDescriptor htd = new HTableDescriptor(tn); + htd.addFamily(new HColumnDescriptor(Bytes.toBytes("d"))); + admin.createTable(htd); + + HTableDescriptor createdHTD = admin.getTableDescriptor(Bytes.toBytes(tn)); + assertEquals(null, createdHTD.getServers()); + } } Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHackyCreateTable.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHackyCreateTable.java?rev=1584180&r1=1584179&r2=1584180&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHackyCreateTable.java (original) +++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHackyCreateTable.java Wed Apr 2 21:03:34 2014 @@ -1,143 +0,0 @@ -/** - * Copyright The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.client; - -import static org.junit.Assert.assertEquals; - -import java.io.IOException; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HServerAddress; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.util.Bytes; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -/** - * Create table only on a set of regionservers instead of whole cluster - * JIRA: https://issues.apache.org/jira/browse/HBASE-10425 - * - */ -public class TestHackyCreateTable { - final Log LOG = LogFactory.getLog(getClass()); - private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - private HBaseAdmin admin; - private static final int NUM_REGION_SERVER = 10; - /** - * Number of regionservers that we will remove in the test from {@link #NUM_REGION_SERVER} - */ - private static final int NUM_REMOVED_RS = 7; - @BeforeClass - public static void setUpBeforeClass() throws Exception { - TEST_UTIL.getConfiguration().set(HConstants.LOAD_BALANCER_IMPL, - "org.apache.hadoop.hbase.master.RegionManager$AssignmentLoadBalancer"); - TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100); - TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250); - TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6); - TEST_UTIL.startMiniCluster(NUM_REGION_SERVER); - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - TEST_UTIL.shutdownMiniCluster(); - } - - @Before - public void setUp() throws Exception { - this.admin = new HBaseAdmin(TEST_UTIL.getConfiguration()); - } - - /** - * Create two tables only on specific number of regionservers (subset of all - * regionservers). Check after the creation whether the regions are assigned - * only to those ones. - * - * @throws IOException - */ - @Test - public void testCreateTableOnRegionServers() throws IOException { - byte[] tableName = Bytes.toBytes("testCreateTableWithRegionsOnServers"); - byte[][] splitKeys = { - new byte[]{1, 1, 1}, - new byte[]{2, 2, 2}, - new byte[]{3, 3, 3}, - new byte[]{4, 4, 4}, - new byte[]{5, 5, 5}, - new byte[]{6, 6, 6}, - new byte[]{7, 7, 7}, - new byte[]{8, 8, 8}, - new byte[]{9, 9, 9}, - }; - HTableDescriptor desc = new HTableDescriptor(tableName); - desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); - List<HServerAddress> allServers = TEST_UTIL.getHBaseCluster().getRegionServers(); - System.out.println("initial number of servers: " + allServers.size()); - // remove some regionservers - for (int i = 0; i < NUM_REMOVED_RS; i++) { - allServers.remove(0); - } - // create a table on the remaining regionservers - // refresh Admin to pickup new configuration about using favored nodes - admin.createTable(desc, splitKeys, allServers); - HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName); - Map<HRegionInfo,HServerAddress> regions = ht.getRegionsInfo(); - int expectedRegions = splitKeys.length + 1; - assertEquals("Tried to create " + expectedRegions + " regions " + - "but only found " + regions.size(), - expectedRegions, regions.size()); - //verify if we use only a subset of the regionservers - Set<HServerAddress> serversForTable = new HashSet<HServerAddress>(); - serversForTable.addAll(regions.values()); - assertEquals(NUM_REGION_SERVER - NUM_REMOVED_RS, allServers.size()); - assertEquals(allServers.size(), serversForTable.size()); - - // create one more table only on allServers - tableName = Bytes.toBytes("testCreateTableWithRegionsOnServers2"); - byte[][] splitKeys2 = { - new byte[]{1, 1, 1}, - new byte[]{2, 2, 2} - }; - desc = new HTableDescriptor(tableName); - desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); - admin.createTable(desc, splitKeys2, allServers); - ht = new HTable(TEST_UTIL.getConfiguration(), tableName); - regions = ht.getRegionsInfo(); - expectedRegions = splitKeys2.length + 1; - assertEquals("Tried to create " + expectedRegions + " regions " + - "but only found " + regions.size(), - expectedRegions, regions.size()); - //verify if we use only a subset of the regionservers - serversForTable = new HashSet<HServerAddress>(); - serversForTable.addAll(regions.values()); - assertEquals(NUM_REGION_SERVER - NUM_REMOVED_RS, allServers.size()); - assertEquals(allServers.size(), serversForTable.size()); - } -} Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/thrift/swift/TestSwiftSerDe.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/thrift/swift/TestSwiftSerDe.java?rev=1584180&r1=1584179&r2=1584180&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/thrift/swift/TestSwiftSerDe.java (original) +++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/thrift/swift/TestSwiftSerDe.java Wed Apr 2 21:03:34 2014 @@ -25,7 +25,11 @@ import static org.junit.Assert.assertTru import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HRegionInfo; @@ -197,6 +201,28 @@ public class TestSwiftSerDe { } @Test + public void testHTDSerDe() throws Exception { + List<HColumnDescriptor> columns = new ArrayList<>(1); + Map<byte[], byte[]> values = new TreeMap<>(Bytes.BYTES_COMPARATOR); + Set<HServerAddress> servers = new HashSet<>(1); + servers.add(new HServerAddress(java.net.InetAddress.getLocalHost().getHostName(),0)); + + columns.add(new HColumnDescriptor(Bytes.toBytes("d"))); + + HTableDescriptor origionalHTD = new HTableDescriptor( + Bytes.toBytes("testSerdeName"), + columns, + values + ); + origionalHTD.setServers(servers); + + byte[] data = Bytes.writeThriftBytes(origionalHTD, HTableDescriptor.class); + HTableDescriptor htdCopy = Bytes.readThriftBytes(data, HTableDescriptor.class); + assertEquals(origionalHTD, htdCopy); + assertEquals(servers, htdCopy.getServers()); + } + + @Test public void testHFileHistogramSerDe() throws Exception { byte [] startRow = Bytes.toBytes("testStartRow"); byte [] endRow = Bytes.toBytes("testEndRow");
