Repository: kylin Updated Branches: refs/heads/yang-m2 [created] 445d458bd
minor change on hbase conf Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/445d458b Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/445d458b Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/445d458b Branch: refs/heads/yang-m2 Commit: 445d458bd97fab187c4de8a9d2bb660610e279e5 Parents: b5e7be9 Author: lidongsjtu <lid...@apache.org> Authored: Mon May 30 17:34:41 2016 +0800 Committer: lidongsjtu <lid...@apache.org> Committed: Mon May 30 17:44:31 2016 +0800 ---------------------------------------------------------------------- build/bin/setenv.sh | 2 +- .../main/java/org/apache/kylin/common/KylinConfigBase.java | 3 +++ .../java/org/apache/kylin/storage/hbase/HBaseConnection.java | 3 +++ .../org/apache/kylin/storage/hbase/steps/CreateHTableJob.java | 7 ++++--- 4 files changed, 11 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/445d458b/build/bin/setenv.sh ---------------------------------------------------------------------- diff --git a/build/bin/setenv.sh b/build/bin/setenv.sh index f98f3c3..d486672 100755 --- a/build/bin/setenv.sh +++ b/build/bin/setenv.sh @@ -19,7 +19,7 @@ # (if your're deploying KYLIN on a powerful server and want to replace the default conservative settings) # uncomment following to for it to take effect -export KYLIN_JVM_SETTINGS="-Xms1024M -Xmx4096M -XX:MaxPermSize=128M -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$KYLIN_HOME/logs/kylin.gc.$$ -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=64M" +export KYLIN_JVM_SETTINGS="-Xms1024M -Xmx4096M -Xss256K -XX:MaxPermSize=128M -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$KYLIN_HOME/logs/kylin.gc.$$ -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=64M" # export KYLIN_JVM_SETTINGS="-Xms16g -Xmx16g -XX:MaxPermSize=512m -XX:NewSize=3g -XX:MaxNewSize=3g -XX:SurvivorRatio=4 -XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode -XX:CMSInitiatingOccupancyFraction=70 -XX:+DisableExplicitGC" # uncomment following to for it to take effect(the values need adjusting to fit your env) http://git-wip-us.apache.org/repos/asf/kylin/blob/445d458b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 1062749..b55c93c 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -646,4 +646,7 @@ abstract public class KylinConfigBase implements Serializable { return Integer.parseInt(getOptional("kylin.job.cubing.inmem.sampling.hll.precision", "14")); } + public String getHBaseMaxConnectionThreads() { + return getOptional("kylin.query.hbase.hconnection.threads.max", "3072"); + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/445d458b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java index e2f4f3a..bf5d5f6 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java @@ -110,6 +110,9 @@ public class HBaseConnection { conf.set(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, "60000"); // conf.set(ScannerCallable.LOG_SCANNER_ACTIVITY, "true"); + // set hbase.hconnection.threads.max to large to unblock region server requests from client threadpool. + conf.set("hbase.hconnection.threads.max", KylinConfig.getInstanceFromEnv().getHBaseMaxConnectionThreads()); + return conf; } http://git-wip-us.apache.org/repos/asf/kylin/blob/445d458b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java index 7a32e37..c5276a3 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java @@ -41,7 +41,6 @@ import org.apache.hadoop.util.ToolRunner; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.Bytes; import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.common.util.Primes; import org.apache.kylin.common.util.ShardingHash; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; @@ -186,7 +185,9 @@ public class CreateHTableJob extends AbstractHadoopJob { if (cubeSegment.isEnableSharding()) {//&& (nRegion > 1)) { //use prime nRegions to help random sharding int original = nRegion; - nRegion = Primes.nextPrime(nRegion);//return 2 for input 1 + if (nRegion == 0) { + nRegion = 1; + } if (nRegion > Short.MAX_VALUE) { logger.info("Too many regions! reduce to " + Short.MAX_VALUE); @@ -301,7 +302,7 @@ public class CreateHTableJob extends AbstractHadoopJob { } int compactionThreshold = Integer.valueOf(hbaseConf.get("hbase.hstore.compactionThreshold", "3")); logger.info("hbase.hstore.compactionThreshold is " + compactionThreshold); - if (hfileSizeMB > 0.0 && hfileSizeMB * compactionThreshold < mbPerRegion) { + if (hfileSizeMB > 0.0 && hfileSizeMB * compactionThreshold < mbPerRegion) { hfileSizeMB = mbPerRegion / compactionThreshold; }
