PHOENIX-2570 Start transaction manager only when needed and reduce number of threads spawned for tests
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/5779f263 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/5779f263 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/5779f263 Branch: refs/heads/4.x-HBase-1.0 Commit: 5779f263c66033f4fb04615b23a6bf943a4afb7f Parents: 5fb51ca Author: Samarth <[email protected]> Authored: Wed Jan 6 15:16:46 2016 -0800 Committer: Samarth <[email protected]> Committed: Wed Jan 6 15:16:46 2016 -0800 ---------------------------------------------------------------------- .../org/apache/phoenix/query/QueryServices.java | 3 ++ .../phoenix/query/QueryServicesOptions.java | 26 ++++++++++++++++ .../java/org/apache/phoenix/query/BaseTest.java | 32 +++++++++----------- .../phoenix/query/QueryServicesTestImpl.java | 12 +++++++- pom.xml | 2 +- 5 files changed, 56 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/5779f263/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java index 5fd7403..c0f4c4f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java @@ -201,6 +201,9 @@ public interface QueryServices extends SQLCloseable { public static final String RUN_RENEW_LEASE_FREQUENCY_INTERVAL_MILLISECONDS = "phoenix.scanner.lease.renew.interval"; public static final String RENEW_LEASE_THRESHOLD_MILLISECONDS = "phoenix.scanner.lease.threshold"; public static final String RENEW_LEASE_THREAD_POOL_SIZE = "phoenix.scanner.lease.pool.size"; + public static final String HCONNECTION_POOL_CORE_SIZE = "hbase.hconnection.threads.core"; + public static final String HCONNECTION_POOL_MAX_SIZE = "hbase.hconnection.threads.max"; + public static final String HTABLE_MAX_THREADS = "hbase.htable.threads.max"; /** * Get executor service used for parallel scans http://git-wip-us.apache.org/repos/asf/phoenix/blob/5779f263/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java index def0a18..efe6a70 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java @@ -616,4 +616,30 @@ public class QueryServicesOptions { config.setBoolean(RENEW_LEASE_ENABLED, enable); return this; } + + public QueryServicesOptions setIndexHandlerCount(int count) { + config.setInt(QueryServices.INDEX_HANDLER_COUNT_ATTRIB, count); + return this; + } + + public QueryServicesOptions setMetadataHandlerCount(int count) { + config.setInt(QueryServices.METADATA_HANDLER_COUNT_ATTRIB, count); + return this; + } + + public QueryServicesOptions setHConnectionPoolCoreSize(int count) { + config.setInt(QueryServices.HCONNECTION_POOL_CORE_SIZE, count); + return this; + } + + public QueryServicesOptions setHConnectionPoolMaxSize(int count) { + config.setInt(QueryServices.HCONNECTION_POOL_MAX_SIZE, count); + return this; + } + + public QueryServicesOptions setMaxThreadsPerHTable(int count) { + config.setInt(QueryServices.HTABLE_MAX_THREADS, count); + return this; + } + } http://git-wip-us.apache.org/repos/asf/phoenix/blob/5779f263/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java index 3f16065..be776b3 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java @@ -112,6 +112,12 @@ import java.util.concurrent.TimeUnit; import javax.annotation.Nonnull; +import co.cask.tephra.TransactionManager; +import co.cask.tephra.TxConstants; +import co.cask.tephra.distributed.TransactionService; +import co.cask.tephra.metrics.TxMetricsCollector; +import co.cask.tephra.persist.InMemoryTransactionStateStorage; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -128,6 +134,7 @@ import org.apache.hadoop.hbase.ipc.PhoenixRpcSchedulerFactory; import org.apache.hadoop.hbase.ipc.RpcControllerFactory; import org.apache.hadoop.hbase.ipc.controller.ServerRpcControllerFactory; import org.apache.hadoop.hbase.master.LoadBalancer; +import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.LocalIndexMerger; import org.apache.hadoop.hbase.regionserver.RSRpcServices; import org.apache.hadoop.hbase.util.Bytes; @@ -163,12 +170,6 @@ import org.junit.rules.TemporaryFolder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import co.cask.tephra.TransactionManager; -import co.cask.tephra.TxConstants; -import co.cask.tephra.distributed.TransactionService; -import co.cask.tephra.metrics.TxMetricsCollector; -import co.cask.tephra.persist.InMemoryTransactionStateStorage; - import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -549,7 +550,6 @@ public abstract class BaseTest { if (!clusterInitialized) { url = setUpTestCluster(config, overrideProps); clusterInitialized = true; - setupTxManager(); } return url; } @@ -573,6 +573,7 @@ public abstract class BaseTest { assertTrue(destroyDriver(driver)); } finally { driver = null; + teardownTxManager(); } } } @@ -594,12 +595,8 @@ public abstract class BaseTest { utility.shutdownMiniCluster(); } } finally { - try { - teardownTxManager(); - } finally { - utility = null; - clusterInitialized = false; - } + utility = null; + clusterInitialized = false; } } } @@ -612,6 +609,9 @@ public abstract class BaseTest { String url = checkClusterInitialized(serverProps); if (driver == null) { driver = initAndRegisterDriver(url, clientProps); + if (clientProps.getBoolean(QueryServices.TRANSACTIONS_ENABLED, QueryServicesOptions.DEFAULT_TRANSACTIONS_ENABLED)) { + setupTxManager(); + } } } @@ -712,9 +712,6 @@ public abstract class BaseTest { conf.setInt("dfs.datanode.handler.count", 2); conf.setInt("ipc.server.read.threadpool.size", 2); conf.setInt("ipc.server.handler.threadpool.size", 2); - conf.setInt("hbase.hconnection.threads.max", 2); - conf.setInt("hbase.hconnection.threads.core", 2); - conf.setInt("hbase.htable.threads.max", 2); conf.setInt("hbase.regionserver.hlog.syncer.count", 2); conf.setInt("hbase.hlog.asyncer.number", 2); conf.setInt("hbase.assignment.zkevent.workers", 5); @@ -734,7 +731,8 @@ public abstract class BaseTest { if (oldDriver != newDriver) { destroyDriver(oldDriver); } - Connection conn = newDriver.connect(url, PropertiesUtil.deepCopy(TEST_PROPERTIES)); + Properties driverProps = PropertiesUtil.deepCopy(TEST_PROPERTIES); + Connection conn = newDriver.connect(url, driverProps); conn.close(); return newDriver; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/5779f263/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java index 037e8e7..8ff9a69 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java @@ -58,6 +58,11 @@ public final class QueryServicesTestImpl extends BaseQueryServicesImpl { private static final boolean DEFAULT_RUN_UPDATE_STATS_ASYNC = false; private static final boolean DEFAULT_COMMIT_STATS_ASYNC = false; public static final boolean DEFAULT_RENEW_LEASE_ENABLED = false; + public static final int DEFAULT_INDEX_HANDLER_COUNT = 5; + public static final int DEFAULT_METADATA_HANDLER_COUNT = 5; + public static final int DEFAULT_HCONNECTION_POOL_CORE_SIZE = 10; + public static final int DEFAULT_HCONNECTION_POOL_MAX_SIZE = 10; + public static final int DEFAULT_HTABLE_MAX_THREADS = 10; /** @@ -99,7 +104,12 @@ public final class QueryServicesTestImpl extends BaseQueryServicesImpl { .setExtraJDBCArguments(DEFAULT_EXTRA_JDBC_ARGUMENTS) .setRunUpdateStatsAsync(DEFAULT_RUN_UPDATE_STATS_ASYNC) .setCommitStatsAsync(DEFAULT_COMMIT_STATS_ASYNC) - .setEnableRenewLease(DEFAULT_RENEW_LEASE_ENABLED); + .setEnableRenewLease(DEFAULT_RENEW_LEASE_ENABLED) + .setIndexHandlerCount(DEFAULT_INDEX_HANDLER_COUNT) + .setMetadataHandlerCount(DEFAULT_METADATA_HANDLER_COUNT) + .setHConnectionPoolCoreSize(DEFAULT_HCONNECTION_POOL_CORE_SIZE) + .setHConnectionPoolMaxSize(DEFAULT_HCONNECTION_POOL_MAX_SIZE) + .setMaxThreadsPerHTable(DEFAULT_HTABLE_MAX_THREADS); } public QueryServicesTestImpl(ReadOnlyProps defaultProps, ReadOnlyProps overrideProps) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/5779f263/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index eebba33..2a9c438 100644 --- a/pom.xml +++ b/pom.xml @@ -130,7 +130,7 @@ <!-- Plugin options --> <numForkedUT>3</numForkedUT> - <numForkedIT>5</numForkedIT> + <numForkedIT>7</numForkedIT> <!-- Set default encoding so multi-byte tests work correctly on the Mac --> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
