Repository: incubator-drill Updated Branches: refs/heads/master f2ebfc6fa -> 379f1f387
DRILL-682: Improve HBase storage engine test execution Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/05e67e7d Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/05e67e7d Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/05e67e7d Branch: refs/heads/master Commit: 05e67e7dfd6b9a83ad3c2495dc56836ad13b27a8 Parents: f2ebfc6 Author: Aditya Kishore <adi...@maprtech.com> Authored: Wed May 7 17:41:14 2014 -0700 Committer: Jacques Nadeau <jacq...@apache.org> Committed: Fri May 9 17:15:01 2014 -0700 ---------------------------------------------------------------------- contrib/storage-hbase/pom.xml | 17 ++- .../store/hbase/HBaseStoragePluginConfig.java | 9 ++ .../org/apache/drill/hbase/BaseHBaseTest.java | 62 ++++++++-- .../drill/hbase/HBaseRecordReaderTest.java | 80 +------------ .../org/apache/drill/hbase/HBaseTestsSuite.java | 118 ++++++++++++++----- .../drill/hbase/TestHBaseFilterPushDown.java | 8 +- .../drill/hbase/TestHBaseProjectPushDown.java | 34 +++++- .../apache/drill/hbase/TestTableGenerator.java | 18 +-- .../src/test/resources/hbase-site.xml | 2 +- .../hbase/hbase_scan_screen_physical.json | 2 +- ...base_scan_screen_physical_column_select.json | 2 +- ...base_scan_screen_physical_family_select.json | 2 +- .../src/test/resources/logback.xml | 20 +++- .../java/org/apache/drill/BaseTestQuery.java | 10 +- 14 files changed, 234 insertions(+), 150 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/05e67e7d/contrib/storage-hbase/pom.xml ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/pom.xml b/contrib/storage-hbase/pom.xml index 38f8a1a..700a91d 100644 --- a/contrib/storage-hbase/pom.xml +++ b/contrib/storage-hbase/pom.xml @@ -28,6 +28,10 @@ <name>contrib/hbase-storage-plugin</name> + <properties> + <hbase.TestSuite>**/HBaseTestsSuite.class</hbase.TestSuite> + </properties> + <dependencies> <dependency> <groupId>org.apache.drill.exec</groupId> @@ -55,7 +59,7 @@ <artifactId>metrics-core</artifactId> <version>2.1.1</version> <scope>test</scope> - </dependency> + </dependency> </dependencies> <build> @@ -64,11 +68,18 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> <configuration> + <includes> + <include>${hbase.TestSuite}</include> + </includes> <systemProperties> <property> <name>hbase.test.root</name> <value>${project.build.directory}/data</value> </property> + <property> + <name>logback.log.dir</name> + <value>${project.build.directory}/surefire-reports</value> + </property> </systemProperties> </configuration> </plugin> @@ -81,7 +92,7 @@ <property> <name>!alt-hadoop</name> </property> - </activation> + </activation> <dependencies> <dependency> <groupId>org.apache.hbase</groupId> @@ -148,7 +159,7 @@ <properties> <alt-hadoop>mapr</alt-hadoop> <rat.excludeSubprojects>true</rat.excludeSubprojects> - </properties> + </properties> <dependencies> <dependency> <groupId>org.apache.hbase</groupId> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/05e67e7d/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePluginConfig.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePluginConfig.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePluginConfig.java index 7eba917..b6ff069 100644 --- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePluginConfig.java +++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePluginConfig.java @@ -26,6 +26,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; +import com.google.common.annotations.VisibleForTesting; @JsonTypeName("hbase") public class HBaseStoragePluginConfig extends StoragePluginConfigBase { @@ -73,4 +74,12 @@ public class HBaseStoragePluginConfig extends StoragePluginConfigBase { public Configuration getHBaseConf() { return hbaseConf; } + + @JsonIgnore + @VisibleForTesting + public void setZookeeperPort(int zookeeperPort) { + this.zookeeperPort = zookeeperPort; + hbaseConf.setInt("hbase.zookeeper.property.clientPort", zookeeperPort); + } + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/05e67e7d/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java index 3037321..0753a4d 100644 --- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java @@ -17,42 +17,82 @@ */ package org.apache.drill.hbase; +import java.io.IOException; import java.util.List; import org.apache.drill.BaseTestQuery; +import org.apache.drill.common.util.FileUtils; +import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.record.RecordBatchLoader; import org.apache.drill.exec.rpc.user.QueryResultBatch; +import org.apache.drill.exec.store.hbase.HBaseStoragePlugin; import org.apache.drill.exec.util.VectorUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.client.HBaseAdmin; import org.junit.AfterClass; import org.junit.Assert; +import org.junit.Before; import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.rules.TestName; + +import com.google.common.base.Charsets; +import com.google.common.io.Files; public class BaseHBaseTest extends BaseTestQuery { - protected static final String TEST_TABLE_1 = "TestTable1"; - protected static HBaseAdmin admin; protected static Configuration conf = HBaseConfiguration.create(); + @Rule public TestName TEST_NAME = new TestName(); + + private int columnWidth = 8; + + @Before + public void printID() throws Exception { + System.out.printf("Running %s#%s\n", getClass().getName(), TEST_NAME.getMethodName()); + } + @BeforeClass public static void setUpBeforeClass() throws Exception { - conf.set("hbase.zookeeper.property.clientPort", "2181"); - admin = new HBaseAdmin(conf); - TestTableGenerator.generateHBaseTable(admin, TEST_TABLE_1, 2, 1000); + /* + * Change the following to HBaseTestsSuite.configure(true, true) if you want + * to test against a mini HBase cluster running within unit test environment + */ + HBaseTestsSuite.configure(false, false); + + HBaseTestsSuite.initCluster(); + HBaseStoragePlugin plugin = (HBaseStoragePlugin) bit.getContext().getStorage().getPlugin("hbase"); + plugin.getConfig().setZookeeperPort(HBaseTestsSuite.getZookeeperPort()); } @AfterClass public static void tearDownAfterClass() throws Exception { - admin.disableTable(TEST_TABLE_1); - admin.deleteTable(TEST_TABLE_1); + HBaseTestsSuite.tearDownCluster(); } - protected void verify(String sql, int expectedRowCount) throws Exception{ - sql = sql.replace("[TABLE_NAME]", TEST_TABLE_1); + protected void setColumnWidth(int columnWidth) { + this.columnWidth = columnWidth; + } + + protected String getPlanText(String planFile, String tableName) throws IOException { + return Files.toString(FileUtils.getResourceAsFile(planFile), Charsets.UTF_8) + .replaceFirst("\"zookeeperPort\".*:.*\\d+", "\"zookeeperPort\" : " + HBaseTestsSuite.getZookeeperPort()) + .replace("[TABLE_NAME]", tableName); + } + + protected void runPhysicalVerifyCount(String planFile, String tableName, int expectedRowCount) throws Exception{ + String physicalPlan = getPlanText(planFile, tableName); + List<QueryResultBatch> results = testPhysicalWithResults(physicalPlan); + printResultAndVerifyRowCount(results, expectedRowCount); + } + + protected void runSQLVerifyCount(String sql, int expectedRowCount) throws Exception{ + sql = sql.replace("[TABLE_NAME]", HBaseTestsSuite.TEST_TABLE_1); List<QueryResultBatch> results = testSqlWithResults(sql); + printResultAndVerifyRowCount(results, expectedRowCount); + } + private void printResultAndVerifyRowCount(List<QueryResultBatch> results, int expectedRowCount) throws SchemaChangeException { int rowCount = 0; RecordBatchLoader loader = new RecordBatchLoader(getAllocator()); for(QueryResultBatch result : results){ @@ -61,7 +101,7 @@ public class BaseHBaseTest extends BaseTestQuery { if (loader.getRecordCount() <= 0) { break; } - VectorUtil.showVectorAccessibleContent(loader, 8); + VectorUtil.showVectorAccessibleContent(loader, columnWidth); loader.clear(); result.release(); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/05e67e7d/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseRecordReaderTest.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseRecordReaderTest.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseRecordReaderTest.java index 078df1f..1462b81 100644 --- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseRecordReaderTest.java +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseRecordReaderTest.java @@ -17,98 +17,26 @@ */ package org.apache.drill.hbase; -import java.util.List; - -import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.exec.client.DrillClient; -import org.apache.drill.exec.pop.PopUnitTestBase; -import org.apache.drill.exec.proto.UserProtos; -import org.apache.drill.exec.record.RecordBatchLoader; -import org.apache.drill.exec.rpc.user.QueryResultBatch; -import org.apache.drill.exec.server.Drillbit; -import org.apache.drill.exec.server.RemoteServiceSet; -import org.apache.drill.exec.util.VectorUtil; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; import org.junit.Test; -public class HBaseRecordReaderTest extends PopUnitTestBase { - static final org.slf4j.Logger logger = - org.slf4j.LoggerFactory.getLogger(HBaseRecordReaderTest.class); - - private static HBaseAdmin admin; - - private static final String tableName = "testTable"; - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - System.out.println("HBaseStorageHandlerTest: setUpBeforeClass()"); - HBaseTestsSuite.setUp(); - admin = new HBaseAdmin(HBaseTestsSuite.getConf()); - TestTableGenerator.generateHBaseTable(admin, tableName, 2, 1000); - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - System.out.println("HBaseStorageHandlerTest: tearDownAfterClass()"); - admin.disableTable(tableName); - admin.deleteTable(tableName); - HBaseTestsSuite.tearDown(); - } +public class HBaseRecordReaderTest extends BaseHBaseTest { @Test public void testLocalDistributed() throws Exception { String planName = "/hbase/hbase_scan_screen_physical.json"; - testHBaseFullEngineRemote(planName, 6); + runPhysicalVerifyCount(planName, HBaseTestsSuite.TEST_TABLE_1, 6); } @Test public void testLocalDistributedColumnSelect() throws Exception { String planName = "/hbase/hbase_scan_screen_physical_column_select.json"; - testHBaseFullEngineRemote(planName, 2); + runPhysicalVerifyCount(planName, HBaseTestsSuite.TEST_TABLE_1, 2); } @Test public void testLocalDistributedFamilySelect() throws Exception { String planName = "/hbase/hbase_scan_screen_physical_family_select.json"; - testHBaseFullEngineRemote(planName, 3); - } - - // specific tests should call this method, - // but it is not marked as a test itself intentionally - public void testHBaseFullEngineRemote( - String planFile, - int records) throws Exception{ - - RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet(); - - DrillConfig config = DrillConfig.create(); - - try(Drillbit bit1 = new Drillbit(config, serviceSet); - DrillClient client = new DrillClient(config, serviceSet.getCoordinator());) { - bit1.run(); - client.connect(); - RecordBatchLoader batchLoader = new RecordBatchLoader(client.getAllocator()); - List<QueryResultBatch> result = client.runQuery( - UserProtos.QueryType.PHYSICAL, - HBaseTestsSuite.getPlanText(planFile)); - - int recordCount = 0; - for (QueryResultBatch b : result) { - batchLoader.load(b.getHeader().getDef(), b.getData()); - VectorUtil.showVectorAccessibleContent(batchLoader); - recordCount += batchLoader.getRecordCount(); - if(b.getData() != null) b.getData().release(); - } - - Assert.assertEquals(records, recordCount); - - } catch (Exception e) { - logger.error(e.getMessage(), e); - throw e; - } + runPhysicalVerifyCount(planName, HBaseTestsSuite.TEST_TABLE_1, 3); } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/05e67e7d/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java index 3039d39..36c31b7 100644 --- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java @@ -19,69 +19,127 @@ package org.apache.drill.hbase; import java.io.IOException; import java.lang.management.ManagementFactory; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.drill.common.util.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.mapred.TestTableInputFormat; +import org.apache.hadoop.hbase.client.HBaseAdmin; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.runner.RunWith; import org.junit.runners.Suite; import org.junit.runners.Suite.SuiteClasses; -import com.google.common.base.Charsets; -import com.google.common.io.Files; - @RunWith(Suite.class) -@SuiteClasses({HBaseRecordReaderTest.class}) +@SuiteClasses({ + HBaseRecordReaderTest.class, + TestHBaseFilterPushDown.class, + TestHBaseProjectPushDown.class}) public class HBaseTestsSuite { - private static final Log LOG = LogFactory.getLog(TestTableInputFormat.class); + private static final Log LOG = LogFactory.getLog(HBaseTestsSuite.class); private static final boolean IS_DEBUG = ManagementFactory.getRuntimeMXBean().getInputArguments().toString().indexOf("-agentlib:jdwp") > 0; + protected static final String TEST_TABLE_1 = "TestTable1"; + private static Configuration conf; + private static HBaseAdmin admin; + private static HBaseTestingUtility UTIL; + private static volatile AtomicInteger initCount = new AtomicInteger(0); + + private static boolean manageHBaseCluster = System.getProperty("drill.hbase.tests.manageHBaseCluster", "true").equalsIgnoreCase("true"); + private static boolean hbaseClusterCreated = false; + + private static boolean createTables = System.getProperty("drill.hbase.tests.createTables", "true").equalsIgnoreCase("true"); + private static boolean tablesCreated = false; + @BeforeClass - public static void setUp() throws Exception { - if (conf == null) { - conf = HBaseConfiguration.create(); - } - conf.set("hbase.zookeeper.property.clientPort", "2181"); - if (IS_DEBUG) { - conf.set("hbase.regionserver.lease.period","1000000"); - } - LOG.info("Starting HBase mini cluster."); - if (UTIL == null) { - UTIL = new HBaseTestingUtility(conf); + public static void initCluster() throws Exception { + if (initCount.get() == 0) { + synchronized (HBaseTestsSuite.class) { + if (initCount.get() == 0) { + conf = HBaseConfiguration.create(); + if (IS_DEBUG) { + conf.set("hbase.regionserver.lease.period","10000000"); + } + + if (manageHBaseCluster) { + LOG.info("Starting HBase mini cluster."); + UTIL = new HBaseTestingUtility(conf); + UTIL.startMiniCluster(); + hbaseClusterCreated = true; + LOG.info("HBase mini cluster started."); + } + + admin = new HBaseAdmin(conf); + + if (createTables || !tablesExist()) { + createTestTables(); + tablesCreated = true; + } + initCount.incrementAndGet(); + return; + } + } } - UTIL.startMiniCluster(); - LOG.info("HBase mini cluster started."); + initCount.incrementAndGet(); } @AfterClass - public static void tearDown() throws Exception { - LOG.info("Shutting down HBase mini cluster."); - UTIL.shutdownMiniCluster(); - LOG.info("HBase mini cluster stopped."); + public static void tearDownCluster() throws Exception { + synchronized (HBaseTestsSuite.class) { + if (initCount.decrementAndGet() == 0) { + if (createTables && tablesCreated) { + cleanupTestTables(); + } + + if (admin != null) { + admin.close(); + } + + if (hbaseClusterCreated) { + LOG.info("Shutting down HBase mini cluster."); + UTIL.shutdownMiniCluster(); + LOG.info("HBase mini cluster stopped."); + } + } + } } public static Configuration getConf() { return conf; } - public static String getPlanText(String planFile) throws IOException { - String text = Files.toString(FileUtils.getResourceAsFile(planFile), Charsets.UTF_8); - return text.replaceFirst("\"zookeeperPort\".*:.*\\d+", "\"zookeeperPort\" : " - + conf.get(HConstants.ZOOKEEPER_CLIENT_PORT)); - } - public static HBaseTestingUtility getHBaseTestingUtility() { return UTIL; } + + private static boolean tablesExist() throws IOException { + return admin.tableExists(TEST_TABLE_1); + } + + private static void createTestTables() throws Exception { + TestTableGenerator.generateHBaseDataset1(admin, TEST_TABLE_1, 2); + } + + private static void cleanupTestTables() throws IOException { + admin.disableTable(TEST_TABLE_1); + admin.deleteTable(TEST_TABLE_1); + } + + public static int getZookeeperPort() { + return getConf().getInt(HConstants.ZOOKEEPER_CLIENT_PORT, 2181); + } + + public static void configure(boolean manageHBaseCluster, boolean createTables) { + HBaseTestsSuite.manageHBaseCluster = manageHBaseCluster; + HBaseTestsSuite.createTables = createTables; + } + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/05e67e7d/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java index 2d72192..6e64c4a 100644 --- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java @@ -17,15 +17,13 @@ */ package org.apache.drill.hbase; -import org.junit.Ignore; import org.junit.Test; -@Ignore // Need to find a way to pass zookeeper port to HBase storage plugin configuration before enabling this test public class TestHBaseFilterPushDown extends BaseHBaseTest { @Test public void testFilterPushDownRowKeyEqual() throws Exception{ - verify("SELECT\n" + runSQLVerifyCount("SELECT\n" + " tableName.*\n" + "FROM\n" + " hbase.`[TABLE_NAME]` tableName\n" @@ -35,7 +33,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest { @Test public void testFilterPushDownRowKeyGreaterThan() throws Exception{ - verify("SELECT\n" + runSQLVerifyCount("SELECT\n" + " tableName.*\n" + "FROM\n" + " hbase.`[TABLE_NAME]` tableName\n" @@ -45,7 +43,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest { @Test public void testFilterPushDownRowKeyLessThanOrEqualTo() throws Exception{ - verify("SELECT\n" + runSQLVerifyCount("SELECT\n" + " tableName.*\n" + "FROM\n" + " hbase.`[TABLE_NAME]` tableName\n" http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/05e67e7d/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java index 0600696..0d91454 100644 --- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java @@ -17,16 +17,42 @@ */ package org.apache.drill.hbase; -import org.junit.Ignore; import org.junit.Test; -@Ignore // Need to find a way to pass zookeeper port to HBase storage plugin configuration before enabling this test public class TestHBaseProjectPushDown extends BaseHBaseTest { @Test public void testRowKeyPushDown() throws Exception{ - verify("SELECT\n" - + "row_key, substring(row_key, 2, 1)*12\n" + runSQLVerifyCount("SELECT\n" + + "row_key\n" + + "FROM\n" + + " hbase.`[TABLE_NAME]` tableName" + , 6); + } + + @Test + public void testColumnWith1RowPushDown() throws Exception{ + runSQLVerifyCount("SELECT\n" + + "f2['c7']\n" + + "FROM\n" + + " hbase.`[TABLE_NAME]` tableName" + , 1); + } + + @Test + public void testRowKeyAndColumnPushDown() throws Exception{ + setColumnWidth(9); + runSQLVerifyCount("SELECT\n" + + "row_key, f['c1']*31 as `f[c1]*31`, f['c2'] as `f['c2']`, 5 as `5`, 'abc' as `'abc'`\n" + + "FROM\n" + + " hbase.`[TABLE_NAME]` tableName" + , 6); + } + + @Test + public void testColumnFamilyPushDown() throws Exception{ + runSQLVerifyCount("SELECT\n" + + "f\n" + "FROM\n" + " hbase.`[TABLE_NAME]` tableName" , 6); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/05e67e7d/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestTableGenerator.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestTableGenerator.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestTableGenerator.java index 76f100e..487c306 100644 --- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestTableGenerator.java +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestTableGenerator.java @@ -32,21 +32,20 @@ public class TestTableGenerator { {'j'}, {'k'}, {'l'}, {'m'}, {'n'}, {'o'}, {'p'}, {'q'}, {'r'}, {'s'}, {'t'}, {'u'}, {'v'}, {'w'}, {'x'}, {'y'}, {'z'} }; - - public static void generateHBaseTable(HBaseAdmin admin, String tableName, int numberRegions, - int recordsPerRegion) throws Exception { + + public static void generateHBaseDataset1(HBaseAdmin admin, String tableName, int numberRegions) throws Exception { if (admin.tableExists(tableName)) { admin.disableTable(tableName); admin.deleteTable(tableName); } - - byte[][] splitKeys = Arrays.copyOfRange(SPLIT_KEYS, 0, numberRegions-1); - + HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(new HColumnDescriptor("f")); desc.addFamily(new HColumnDescriptor("f2")); - admin.createTable(desc, splitKeys); + admin.createTable(desc, Arrays.copyOfRange(SPLIT_KEYS, 0, numberRegions-1)); + HTable table = new HTable(admin.getConfiguration(), tableName); + Put p = new Put("a1".getBytes()); p.add("f".getBytes(), "c1".getBytes(), "1".getBytes()); p.add("f".getBytes(), "c2".getBytes(), "2".getBytes()); @@ -55,6 +54,7 @@ public class TestTableGenerator { p.add("f".getBytes(), "c5".getBytes(), "5".getBytes()); p.add("f".getBytes(), "c6".getBytes(), "6".getBytes()); table.put(p); + p = new Put("a2".getBytes()); p.add("f".getBytes(), "c1".getBytes(), "1".getBytes()); p.add("f".getBytes(), "c2".getBytes(), "2".getBytes()); @@ -63,6 +63,7 @@ public class TestTableGenerator { p.add("f".getBytes(), "c5".getBytes(), "5".getBytes()); p.add("f".getBytes(), "c6".getBytes(), "6".getBytes()); table.put(p); + p = new Put("a3".getBytes()); p.add("f".getBytes(), "c1".getBytes(), "1".getBytes()); p.add("f".getBytes(), "c3".getBytes(), "2".getBytes()); @@ -71,6 +72,7 @@ public class TestTableGenerator { p.add("f".getBytes(), "c8".getBytes(), "5".getBytes()); p.add("f".getBytes(), "c9".getBytes(), "6".getBytes()); table.put(p); + p = new Put("b4".getBytes()); p.add("f".getBytes(), "c1".getBytes(), "1".getBytes()); p.add("f2".getBytes(), "c2".getBytes(), "2".getBytes()); @@ -79,6 +81,7 @@ public class TestTableGenerator { p.add("f".getBytes(), "c5".getBytes(), "5".getBytes()); p.add("f2".getBytes(), "c6".getBytes(), "6".getBytes()); table.put(p); + p = new Put("b5".getBytes()); p.add("f2".getBytes(), "c1".getBytes(), "1".getBytes()); p.add("f".getBytes(), "c2".getBytes(), "2".getBytes()); @@ -87,6 +90,7 @@ public class TestTableGenerator { p.add("f2".getBytes(), "c5".getBytes(), "5".getBytes()); p.add("f".getBytes(), "c6".getBytes(), "6".getBytes()); table.put(p); + p = new Put("b6".getBytes()); p.add("f".getBytes(), "c1".getBytes(), "1".getBytes()); p.add("f2".getBytes(), "c3".getBytes(), "2".getBytes()); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/05e67e7d/contrib/storage-hbase/src/test/resources/hbase-site.xml ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/test/resources/hbase-site.xml b/contrib/storage-hbase/src/test/resources/hbase-site.xml index 77056b1..54425d4 100644 --- a/contrib/storage-hbase/src/test/resources/hbase-site.xml +++ b/contrib/storage-hbase/src/test/resources/hbase-site.xml @@ -118,7 +118,7 @@ </property> <property> <name>hbase.zookeeper.property.clientPort</name> - <value>21818</value> + <value>2181</value> <description>Property from ZooKeeper's config zoo.cfg. The port at which the clients will connect. </description> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/05e67e7d/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical.json ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical.json b/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical.json index 60d314b..17e17e4 100644 --- a/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical.json +++ b/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical.json @@ -10,7 +10,7 @@ pop : "hbase-scan", @id : 1, hbaseScanSpec : { - tableName : "testTable" + tableName : "[TABLE_NAME]" }, storage: { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/05e67e7d/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical_column_select.json ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical_column_select.json b/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical_column_select.json index f44f568..c64dc97 100644 --- a/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical_column_select.json +++ b/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical_column_select.json @@ -10,7 +10,7 @@ pop : "hbase-scan", @id : 1, hbaseScanSpec : { - tableName : "testTable" + tableName : "[TABLE_NAME]" }, storage: { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/05e67e7d/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical_family_select.json ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical_family_select.json b/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical_family_select.json index c68fef3..ce027a0 100644 --- a/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical_family_select.json +++ b/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical_family_select.json @@ -10,7 +10,7 @@ pop : "hbase-scan", @id : 1, hbaseScanSpec : { - tableName : "testTable" + tableName : "[TABLE_NAME]" }, storage: { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/05e67e7d/contrib/storage-hbase/src/test/resources/logback.xml ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/test/resources/logback.xml b/contrib/storage-hbase/src/test/resources/logback.xml index e8d40d4..b89ceaa 100644 --- a/contrib/storage-hbase/src/test/resources/logback.xml +++ b/contrib/storage-hbase/src/test/resources/logback.xml @@ -16,6 +16,8 @@ limitations under the License. --> <configuration> + <timestamp key="bySecond" datePattern="yyyyMMdd'T'HHmmss"/> + <appender name="SOCKET" class="de.huxhorn.lilith.logback.appender.ClassicMultiplexSocketAppender"> <Compressing>true</Compressing> @@ -24,10 +26,18 @@ <RemoteHosts>localhost</RemoteHosts> </appender> + <appender name="FILE" class="ch.qos.logback.core.FileAppender"> + <!-- The property 'logback.log.dir' is defined in pom.xml --> + <file>${logback.log.dir:-./target/surefire-reports}/hbase-tests-${bySecond}.log</file> + <append>false</append> + <encoder> + <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern> + </encoder> + </appender> + <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> <encoder> - <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - </pattern> + <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern> </encoder> </appender> @@ -41,14 +51,14 @@ <appender-ref ref="SOCKET" /> </logger> - <logger name="org.apache.hadoop.hbase" additivity="false"> + <logger name="org.apache.hadoop" additivity="false"> <level value="info" /> - <appender-ref ref="STDOUT" /> + <appender-ref ref="FILE" /> </logger> <root> <level value="error" /> <appender-ref ref="STDOUT" /> - </root> + </root> </configuration> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/05e67e7d/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java index 86a37ca..0e5963a 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java +++ b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java @@ -58,11 +58,11 @@ public class BaseTestQuery extends ExecTest{ } }; - static DrillClient client; - static Drillbit bit; - static RemoteServiceSet serviceSet; - static DrillConfig config; - static QuerySubmitter submitter = new QuerySubmitter(); + protected static DrillClient client; + protected static Drillbit bit; + protected static RemoteServiceSet serviceSet; + protected static DrillConfig config; + protected static QuerySubmitter submitter = new QuerySubmitter(); static void resetClientAndBit() throws Exception{ closeClient();
