KYLIN-2413 update test conf files
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/87d10394 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/87d10394 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/87d10394 Branch: refs/heads/KYLIN-2361 Commit: 87d103948fdf8951941c9a7fd11213dd569f8aa5 Parents: eb73da7 Author: lidongsjtu <lid...@apache.org> Authored: Mon Jan 23 13:12:36 2017 +0800 Committer: lidongsjtu <lid...@apache.org> Committed: Mon Jan 23 13:18:24 2017 +0800 ---------------------------------------------------------------------- dev-support/sync_hbase_cdh_branches.sh | 15 +-- dev-support/test_all_against_hdp_2_2_4_2_2.sh | 25 ---- dev-support/test_all_against_hdp_2_4_0_0_169.sh | 25 ++++ .../sandbox/capacity-scheduler.xml | 17 +-- examples/test_case_data/sandbox/core-site.xml | 35 ++++- .../test_case_data/sandbox/hadoop-policy.xml | 2 +- .../test_case_data/sandbox/hbase-policy.xml | 2 +- examples/test_case_data/sandbox/hbase-site.xml | 100 +++++++++++++-- examples/test_case_data/sandbox/hdfs-site.xml | 84 +++++++----- examples/test_case_data/sandbox/hive-site.xml | 89 ++++++++----- examples/test_case_data/sandbox/mapred-site.xml | 57 ++++++--- examples/test_case_data/sandbox/yarn-site.xml | 127 ++++++++++++++++--- 12 files changed, 404 insertions(+), 174 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/dev-support/sync_hbase_cdh_branches.sh ---------------------------------------------------------------------- diff --git a/dev-support/sync_hbase_cdh_branches.sh b/dev-support/sync_hbase_cdh_branches.sh index 472f402..592053a 100755 --- a/dev-support/sync_hbase_cdh_branches.sh +++ b/dev-support/sync_hbase_cdh_branches.sh @@ -37,24 +37,17 @@ trap 'error ${LINENO} ${?}' ERR # ============================================================================ git fetch apache -git checkout apache/$base-hbase1.x -git format-patch -1 -git checkout apache/$base-cdh5.7 +git checkout apache/$base-hbase0.98 git format-patch -1 git checkout apache/$base git checkout -b tmp git reset apache/$base --hard -git am -3 --ignore-whitespace 0001-KYLIN-1528-Create-a-branch-for-v1.5-with-HBase-1.x-A.patch -mvn clean compile -DskipTests -git push apache tmp:$base-hbase1.x -f -rm 0001-KYLIN-1528-Create-a-branch-for-v1.5-with-HBase-1.x-A.patch - -git am -3 --ignore-whitespace 0001-KYLIN-1672-support-kylin-on-cdh-5.7.patch +git am -3 --ignore-whitespace 0001-KYLIN-2307-Create-a-branch-for-master-with-HBase-0.9.patch mvn clean compile -DskipTests -git push apache tmp:$base-cdh5.7 -f -rm 0001-KYLIN-1672-support-kylin-on-cdh-5.7.patch +git push apache tmp:$base-hbase0.98 -f +rm 0001-KYLIN-2307-Create-a-branch-for-master-with-HBase-0.9.patch # clean up git checkout master http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/dev-support/test_all_against_hdp_2_2_4_2_2.sh ---------------------------------------------------------------------- diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh b/dev-support/test_all_against_hdp_2_2_4_2_2.sh deleted file mode 100755 index f7780dd..0000000 --- a/dev-support/test_all_against_hdp_2_2_4_2_2.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -dir=$(dirname ${0}) -cd ${dir} -cd .. - -mvn clean install -DskipTests 2>&1 | tee mci.log -mvn verify -Dhdp.version=${HDP_VERSION:-"2.2.4.2-2"} -fae 2>&1 | tee mvnverify.log http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/dev-support/test_all_against_hdp_2_4_0_0_169.sh ---------------------------------------------------------------------- diff --git a/dev-support/test_all_against_hdp_2_4_0_0_169.sh b/dev-support/test_all_against_hdp_2_4_0_0_169.sh new file mode 100755 index 0000000..2a3d24b --- /dev/null +++ b/dev-support/test_all_against_hdp_2_4_0_0_169.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +dir=$(dirname ${0}) +cd ${dir} +cd .. + +mvn clean install -DskipTests 2>&1 | tee mci.log +mvn verify -Dhdp.version=${HDP_VERSION:-"2.4.0.0-169"} -fae 2>&1 | tee mvnverify.log http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/examples/test_case_data/sandbox/capacity-scheduler.xml ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/capacity-scheduler.xml b/examples/test_case_data/sandbox/capacity-scheduler.xml index 7cb985c..e042aa5 100644 --- a/examples/test_case_data/sandbox/capacity-scheduler.xml +++ b/examples/test_case_data/sandbox/capacity-scheduler.xml @@ -47,16 +47,6 @@ </property> <property> - <name>yarn.scheduler.capacity.root.accessible-node-labels.default.capacity</name> - <value>-1</value> - </property> - - <property> - <name>yarn.scheduler.capacity.root.accessible-node-labels.default.maximum-capacity</name> - <value>-1</value> - </property> - - <property> <name>yarn.scheduler.capacity.root.acl_administer_queue</name> <value>*</value> </property> @@ -67,11 +57,6 @@ </property> <property> - <name>yarn.scheduler.capacity.root.default-node-label-expression</name> - <value></value> - </property> - - <property> <name>yarn.scheduler.capacity.root.default.acl_administer_jobs</name> <value>*</value> </property> @@ -111,4 +96,4 @@ <value>default</value> </property> -</configuration> +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/examples/test_case_data/sandbox/core-site.xml ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/core-site.xml b/examples/test_case_data/sandbox/core-site.xml index 9aa588c..7660a7e 100644 --- a/examples/test_case_data/sandbox/core-site.xml +++ b/examples/test_case_data/sandbox/core-site.xml @@ -19,7 +19,6 @@ <property> <name>fs.defaultFS</name> <value>hdfs://sandbox.hortonworks.com:8020</value> - <final>true</final> </property> <property> @@ -39,7 +38,7 @@ <property> <name>hadoop.proxyuser.falcon.groups</name> - <value>users</value> + <value>*</value> </property> <property> @@ -49,7 +48,7 @@ <property> <name>hadoop.proxyuser.hbase.groups</name> - <value>users</value> + <value>*</value> </property> <property> @@ -68,13 +67,23 @@ </property> <property> + <name>hadoop.proxyuser.hdfs.groups</name> + <value>*</value> + </property> + + <property> + <name>hadoop.proxyuser.hdfs.hosts</name> + <value>*</value> + </property> + + <property> <name>hadoop.proxyuser.hive.groups</name> - <value>users</value> + <value>*</value> </property> <property> <name>hadoop.proxyuser.hive.hosts</name> - <value>*</value> + <value>sandbox.hortonworks.com</value> </property> <property> @@ -123,8 +132,15 @@ </property> <property> + <name>hadoop.security.key.provider.path</name> + <value></value> + </property> + + <property> <name>io.compression.codecs</name> - <value>org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec</value> + <value> + org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec + </value> </property> <property> @@ -162,4 +178,9 @@ <value>false</value> </property> -</configuration> + <property> + <name>net.topology.script.file.name</name> + <value>/etc/hadoop/conf/topology_script.py</value> + </property> + +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/examples/test_case_data/sandbox/hadoop-policy.xml ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/hadoop-policy.xml b/examples/test_case_data/sandbox/hadoop-policy.xml index a590979..20b894e 100644 --- a/examples/test_case_data/sandbox/hadoop-policy.xml +++ b/examples/test_case_data/sandbox/hadoop-policy.xml @@ -71,4 +71,4 @@ <value>hadoop</value> </property> -</configuration> +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/examples/test_case_data/sandbox/hbase-policy.xml ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/hbase-policy.xml b/examples/test_case_data/sandbox/hbase-policy.xml index 178b559..e0f2da6 100644 --- a/examples/test_case_data/sandbox/hbase-policy.xml +++ b/examples/test_case_data/sandbox/hbase-policy.xml @@ -31,4 +31,4 @@ <value>*</value> </property> -</configuration> +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/examples/test_case_data/sandbox/hbase-site.xml ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml index 734908e..568de2e 100644 --- a/examples/test_case_data/sandbox/hbase-site.xml +++ b/examples/test_case_data/sandbox/hbase-site.xml @@ -22,8 +22,33 @@ </property> <property> + <name>hbase.bucketcache.ioengine</name> + <value></value> + </property> + + <property> + <name>hbase.bucketcache.percentage.in.combinedcache</name> + <value></value> + </property> + + <property> + <name>hbase.bucketcache.size</name> + <value></value> + </property> + + <property> + <name>hbase.bulkload.staging.dir</name> + <value>/apps/hbase/staging</value> + </property> + + <property> <name>hbase.client.keyvalue.maxsize</name> - <value>10485760</value> + <value>1048576</value> + </property> + + <property> + <name>hbase.client.retries.number</name> + <value>35</value> </property> <property> @@ -38,12 +63,19 @@ <property> <name>hbase.coprocessor.master.classes</name> - <value>com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor</value> + <value>org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor</value> </property> <property> <name>hbase.coprocessor.region.classes</name> - <value>com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor</value> + <value> + org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint,org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor + </value> + </property> + + <property> + <name>hbase.coprocessor.regionserver.classes</name> + <value></value> </property> <property> @@ -87,6 +119,11 @@ </property> <property> + <name>hbase.hstore.compaction.max</name> + <value>10</value> + </property> + + <property> <name>hbase.hstore.compactionThreshold</name> <value>3</value> </property> @@ -103,32 +140,42 @@ <property> <name>hbase.master.info.port</name> - <value>60010</value> + <value>16010</value> </property> <property> <name>hbase.master.port</name> - <value>60000</value> + <value>16000</value> </property> <property> - <name>hbase.regionserver.global.memstore.lowerLimit</name> - <value>0.38</value> + <name>hbase.region.server.rpc.scheduler.factory.class</name> + <value></value> </property> <property> - <name>hbase.regionserver.global.memstore.upperLimit</name> + <name>hbase.regionserver.global.memstore.size</name> <value>0.4</value> </property> <property> <name>hbase.regionserver.handler.count</name> - <value>60</value> + <value>30</value> </property> <property> <name>hbase.regionserver.info.port</name> - <value>60030</value> + <value>16030</value> + </property> + + <property> + <name>hbase.regionserver.port</name> + <value>16020</value> + </property> + + <property> + <name>hbase.regionserver.wal.codec</name> + <value>org.apache.hadoop.hbase.regionserver.wal.WALCellCodec</value> </property> <property> @@ -137,11 +184,26 @@ </property> <property> + <name>hbase.rpc.controllerfactory.class</name> + <value></value> + </property> + + <property> + <name>hbase.rpc.engine</name> + <value>org.apache.hadoop.hbase.ipc.SecureRpcEngine</value> + </property> + + <property> <name>hbase.rpc.protection</name> <value>PRIVACY</value> </property> <property> + <name>hbase.rpc.timeout</name> + <value>90000</value> + </property> + + <property> <name>hbase.security.authentication</name> <value>simple</value> </property> @@ -158,7 +220,7 @@ <property> <name>hbase.tmp.dir</name> - <value>/hadoop/hbase</value> + <value>/tmp/hbase-${user.name}</value> </property> <property> @@ -178,12 +240,22 @@ <property> <name>hfile.block.cache.size</name> - <value>0.40</value> + <value>0.4</value> + </property> + + <property> + <name>phoenix.functions.allowUserDefinedFunctions</name> + <value></value> + </property> + + <property> + <name>phoenix.query.timeoutMs</name> + <value>60000</value> </property> <property> <name>zookeeper.session.timeout</name> - <value>30000</value> + <value>60000</value> </property> <property> @@ -191,4 +263,4 @@ <value>/hbase-unsecure</value> </property> -</configuration> +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/examples/test_case_data/sandbox/hdfs-site.xml ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/hdfs-site.xml b/examples/test_case_data/sandbox/hdfs-site.xml index 1175fff..1d9040a 100644 --- a/examples/test_case_data/sandbox/hdfs-site.xml +++ b/examples/test_case_data/sandbox/hdfs-site.xml @@ -18,12 +18,7 @@ <property> <name>dfs.block.access.token.enable</name> - <value>false</value> - </property> - - <property> - <name>dfs.block.size</name> - <value>34217472</value> + <value>true</value> </property> <property> @@ -47,11 +42,21 @@ </property> <property> + <name>dfs.client.retry.policy.enabled</name> + <value>false</value> + </property> + + <property> <name>dfs.cluster.administrators</name> <value>hdfs</value> </property> <property> + <name>dfs.content-summary.limit</name> + <value>5000</value> + </property> + + <property> <name>dfs.datanode.address</name> <value>0.0.0.0:50010</value> </property> @@ -64,7 +69,6 @@ <property> <name>dfs.datanode.data.dir</name> <value>/hadoop/hdfs/data</value> - <final>true</final> </property> <property> @@ -80,7 +84,6 @@ <property> <name>dfs.datanode.failed.volumes.tolerated</name> <value>0</value> - <final>true</final> </property> <property> @@ -104,13 +107,18 @@ </property> <property> - <name>dfs.datanode.max.xcievers</name> - <value>1024</value> + <name>dfs.domain.socket.path</name> + <value>/var/lib/hadoop-hdfs/dn_socket</value> </property> <property> - <name>dfs.domain.socket.path</name> - <value>/var/lib/hadoop-hdfs/dn_socket</value> + <name>dfs.encrypt.data.transfer.cipher.suites</name> + <value>AES/CTR/NoPadding</value> + </property> + + <property> + <name>dfs.encryption.key.provider.uri</name> + <value></value> </property> <property> @@ -150,7 +158,12 @@ <property> <name>dfs.namenode.accesstime.precision</name> - <value>3600000</value> + <value>0</value> + </property> + + <property> + <name>dfs.namenode.audit.log.async</name> + <value>true</value> </property> <property> @@ -184,6 +197,11 @@ </property> <property> + <name>dfs.namenode.fslock.fair</name> + <value>false</value> + </property> + + <property> <name>dfs.namenode.handler.count</name> <value>100</value> </property> @@ -191,7 +209,6 @@ <property> <name>dfs.namenode.http-address</name> <value>sandbox.hortonworks.com:50070</value> - <final>true</final> </property> <property> @@ -200,9 +217,13 @@ </property> <property> + <name>dfs.namenode.inode.attributes.provider.class</name> + <value>org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer</value> + </property> + + <property> <name>dfs.namenode.name.dir</name> <value>/hadoop/hdfs/namenode</value> - <final>true</final> </property> <property> @@ -211,8 +232,13 @@ </property> <property> + <name>dfs.namenode.rpc-address</name> + <value>sandbox.hortonworks.com:8020</value> + </property> + + <property> <name>dfs.namenode.safemode.threshold-pct</name> - <value>1.0f</value> + <value>0.999</value> </property> <property> @@ -236,16 +262,6 @@ </property> <property> - <name>dfs.nfs.exports.allowed.hosts</name> - <value>* rw</value> - </property> - - <property> - <name>dfs.nfs3.dump.dir</name> - <value>/tmp/.hdfs-nfs</value> - </property> - - <property> <name>dfs.permissions.enabled</name> <value>true</value> </property> @@ -257,7 +273,7 @@ <property> <name>dfs.replication</name> - <value>1</value> + <value>3</value> </property> <property> @@ -268,13 +284,11 @@ <property> <name>dfs.support.append</name> <value>true</value> - <final>true</final> </property> <property> <name>dfs.webhdfs.enabled</name> <value>true</value> - <final>true</final> </property> <property> @@ -282,4 +296,14 @@ <value>022</value> </property> -</configuration> + <property> + <name>nfs.exports.allowed.hosts</name> + <value>* rw</value> + </property> + + <property> + <name>nfs.file.dump.dir</name> + <value>/tmp/.hdfs-nfs</value> + </property> + +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/examples/test_case_data/sandbox/hive-site.xml ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/hive-site.xml b/examples/test_case_data/sandbox/hive-site.xml index 1e78107..a8c210e 100644 --- a/examples/test_case_data/sandbox/hive-site.xml +++ b/examples/test_case_data/sandbox/hive-site.xml @@ -22,11 +22,46 @@ </property> <property> + <name>atlas.cluster.name</name> + <value>Sandbox</value> + </property> + + <property> + <name>atlas.hook.hive.maxThreads</name> + <value>1</value> + </property> + + <property> + <name>atlas.hook.hive.minThreads</name> + <value>1</value> + </property> + + <property> + <name>atlas.hook.hive.synchronous</name> + <value>true</value> + </property> + + <property> + <name>atlas.rest.address</name> + <value>http://sandbox.hortonworks.com:21000</value> + </property> + + <property> + <name>datanucleus.autoCreateSchema</name> + <value>false</value> + </property> + + <property> <name>datanucleus.cache.level2.type</name> <value>none</value> </property> <property> + <name>datanucleus.fixedDatastore</name> + <value>true</value> + </property> + + <property> <name>hive.auto.convert.join</name> <value>true</value> </property> @@ -38,7 +73,7 @@ <property> <name>hive.auto.convert.join.noconditionaltask.size</name> - <value>1000000000</value> + <value>357913941</value> </property> <property> @@ -127,6 +162,16 @@ </property> <property> + <name>hive.default.fileformat</name> + <value>TextFile</value> + </property> + + <property> + <name>hive.default.fileformat.managed</name> + <value>TextFile</value> + </property> + + <property> <name>hive.enforce.bucketing</name> <value>true</value> </property> @@ -162,11 +207,6 @@ </property> <property> - <name>hive.exec.failure.hooks</name> - <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value> - </property> - - <property> <name>hive.exec.max.created.files</name> <value>100000</value> </property> @@ -197,6 +237,11 @@ </property> <property> + <name>hive.exec.orc.encoding.strategy</name> + <value>SPEED</value> + </property> + + <property> <name>hive.exec.parallel</name> <value>false</value> </property> @@ -207,16 +252,6 @@ </property> <property> - <name>hive.exec.post.hooks</name> - <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value> - </property> - - <property> - <name>hive.exec.pre.hooks</name> - <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value> - </property> - - <property> <name>hive.exec.reducers.bytes.per.reducer</name> <value>67108864</value> </property> @@ -262,11 +297,6 @@ </property> <property> - <name>hive.heapsize</name> - <value>250</value> - </property> - - <property> <name>hive.limit.optimize.enable</name> <value>true</value> </property> @@ -478,7 +508,7 @@ <property> <name>hive.prewarm.numcontainers</name> - <value>10</value> + <value>3</value> </property> <property> @@ -488,7 +518,7 @@ <property> <name>hive.security.authorization.enabled</name> - <value>false</value> + <value>true</value> </property> <property> @@ -508,7 +538,7 @@ <property> <name>hive.security.metastore.authorization.manager</name> - <value>org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider,org.apache.hadoop.hive.ql.security.authorization.MetaStoreAuthzAPIAuthorizerEmbedOnly</value> + <value>org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider</value> </property> <property> @@ -533,12 +563,7 @@ <property> <name>hive.server2.enable.doAs</name> - <value>true</value> - </property> - - <property> - <name>hive.server2.enable.impersonation</name> - <value>true</value> + <value>false</value> </property> <property> @@ -548,7 +573,7 @@ <property> <name>hive.server2.logging.operation.log.location</name> - <value>${system:java.io.tmpdir}/${system:user.name}/operation_logs</value> + <value>/tmp/hive/operation_logs</value> </property> <property> @@ -653,7 +678,7 @@ <property> <name>hive.tez.container.size</name> - <value>250</value> + <value>1024</value> </property> <property> http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/examples/test_case_data/sandbox/mapred-site.xml ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml index e90f594..be470f9 100644 --- a/examples/test_case_data/sandbox/mapred-site.xml +++ b/examples/test_case_data/sandbox/mapred-site.xml @@ -18,7 +18,7 @@ <property> <name>io.sort.mb</name> - <value>128</value> + <value>64</value> </property> <property> @@ -27,13 +27,13 @@ </property> <property> - <name>mapreduce.map.memory.mb</name> - <value>512</value> + <name>mapred.job.map.memory.mb</name> + <value>250</value> </property> <property> - <name>mapreduce.reduce.memory.mb</name> - <value>512</value> + <name>mapred.job.reduce.memory.mb</name> + <value>250</value> </property> <property> @@ -48,7 +48,9 @@ <property> <name>mapreduce.admin.user.env</name> - <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64</value> + <value> + LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64 + </value> </property> <property> @@ -58,7 +60,9 @@ <property> <name>mapreduce.application.classpath</name> - <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/usr/hdp/${hdp.version}/hadoop/lib/snappy-java-1.0.4.1.jar:/etc/hadoop/conf/secure</value> + <value> + $PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:$PWD/mr-framework/hadoop/share/hadoop/tools/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure + </value> </property> <property> @@ -77,14 +81,18 @@ </property> <property> + <name>mapreduce.job.counters.max</name> + <value>130</value> + </property> + + <property> <name>mapreduce.job.emit-timeline-data</name> <value>false</value> </property> - <!--the default value on hdp is 0.05, however for test environments we need to be conservative on resource --> <property> <name>mapreduce.job.reduce.slowstart.completedmaps</name> - <value>1</value> + <value>0.05</value> </property> <property> @@ -108,13 +116,28 @@ </property> <property> + <name>mapreduce.jobhistory.recovery.enable</name> + <value>true</value> + </property> + + <property> + <name>mapreduce.jobhistory.recovery.store.class</name> + <value>org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService</value> + </property> + + <property> + <name>mapreduce.jobhistory.recovery.store.leveldb.path</name> + <value>/hadoop/mapreduce/jhs</value> + </property> + + <property> <name>mapreduce.jobhistory.webapp.address</name> <value>sandbox.hortonworks.com:19888</value> </property> <property> <name>mapreduce.map.java.opts</name> - <value>-Xmx512m</value> + <value>-Xmx1228m</value> </property> <property> @@ -124,7 +147,7 @@ <property> <name>mapreduce.map.memory.mb</name> - <value>512</value> + <value>1536</value> </property> <property> @@ -159,7 +182,7 @@ <property> <name>mapreduce.reduce.java.opts</name> - <value>-Xmx200m</value> + <value>-Xmx1638m</value> </property> <property> @@ -169,7 +192,7 @@ <property> <name>mapreduce.reduce.memory.mb</name> - <value>512</value> + <value>2048</value> </property> <property> @@ -219,7 +242,7 @@ <property> <name>mapreduce.task.io.sort.mb</name> - <value>128</value> + <value>859</value> </property> <property> @@ -234,7 +257,7 @@ <property> <name>yarn.app.mapreduce.am.command-opts</name> - <value>-Xmx512m</value> + <value>-Xmx819m -Dhdp.version=${hdp.version}</value> </property> <property> @@ -244,7 +267,7 @@ <property> <name>yarn.app.mapreduce.am.resource.mb</name> - <value>512</value> + <value>1024</value> </property> <property> @@ -252,4 +275,4 @@ <value>/user</value> </property> -</configuration> +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/examples/test_case_data/sandbox/yarn-site.xml ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/yarn-site.xml b/examples/test_case_data/sandbox/yarn-site.xml index 8256158..ebdf44a 100644 --- a/examples/test_case_data/sandbox/yarn-site.xml +++ b/examples/test_case_data/sandbox/yarn-site.xml @@ -18,7 +18,7 @@ <property> <name>hadoop.registry.rm.enabled</name> - <value>false</value> + <value>true</value> </property> <property> @@ -28,22 +28,29 @@ <property> <name>yarn.acl.enable</name> - <value>false</value> + <value>true</value> </property> <property> <name>yarn.admin.acl</name> - <value></value> + <value>*</value> </property> <property> <name>yarn.application.classpath</name> - <value>$HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*</value> + <value> + $HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/* + </value> + </property> + + <property> + <name>yarn.authorization-provider</name> + <value>org.apache.ranger.authorization.yarn.authorizer.RangerYarnAuthorizer</value> </property> <property> <name>yarn.client.nodemanager-connect.max-wait-ms</name> - <value>60000</value> + <value>120000</value> </property> <property> @@ -72,6 +79,11 @@ </property> <property> + <name>yarn.node-labels.enabled</name> + <value>false</value> + </property> + + <property> <name>yarn.node-labels.fs-store.retry-policy-spec</name> <value>2000, 500</value> </property> @@ -82,11 +94,6 @@ </property> <property> - <name>yarn.node-labels.manager-class</name> - <value>org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager</value> - </property> - - <property> <name>yarn.nodemanager.address</name> <value>0.0.0.0:45454</value> </property> @@ -98,7 +105,7 @@ <property> <name>yarn.nodemanager.aux-services</name> - <value>mapreduce_shuffle</value> + <value>mapreduce_shuffle,spark_shuffle</value> </property> <property> @@ -107,6 +114,11 @@ </property> <property> + <name>yarn.nodemanager.aux-services.spark_shuffle.class</name> + <value>org.apache.spark.network.yarn.YarnShuffleService</value> + </property> + + <property> <name>yarn.nodemanager.bind-host</name> <value>0.0.0.0</value> </property> @@ -148,7 +160,7 @@ <property> <name>yarn.nodemanager.health-checker.script.timeout-ms</name> - <value>60000</value> + <value>120000</value> </property> <property> @@ -243,12 +255,12 @@ <property> <name>yarn.nodemanager.resource.memory-mb</name> - <value>9216</value> + <value>7168</value> </property> <property> <name>yarn.nodemanager.resource.percentage-physical-cpu-limit</name> - <value>100</value> + <value>80</value> </property> <property> @@ -337,6 +349,11 @@ </property> <property> + <name>yarn.resourcemanager.scheduler.monitor.enable</name> + <value>false</value> + </property> + + <property> <name>yarn.resourcemanager.state-store.max-completed-applications</name> <value>${yarn.resourcemanager.max-completed-applications}</value> </property> @@ -368,7 +385,7 @@ <property> <name>yarn.resourcemanager.webapp.https.address</name> - <value>localhost:8090</value> + <value>sandbox.hortonworks.com:8090</value> </property> <property> @@ -408,7 +425,7 @@ <property> <name>yarn.resourcemanager.zk-address</name> - <value>localhost:2181</value> + <value>sandbox.hortonworks.com:2181</value> </property> <property> @@ -433,12 +450,22 @@ <property> <name>yarn.scheduler.maximum-allocation-mb</name> - <value>9216</value> + <value>7168</value> + </property> + + <property> + <name>yarn.scheduler.maximum-allocation-vcores</name> + <value>3</value> </property> <property> <name>yarn.scheduler.minimum-allocation-mb</name> - <value>1536</value> + <value>1024</value> + </property> + + <property> + <name>yarn.scheduler.minimum-allocation-vcores</name> + <value>1</value> </property> <property> @@ -467,6 +494,41 @@ </property> <property> + <name>yarn.timeline-service.entity-group-fs-store.active-dir</name> + <value>/ats/active/</value> + </property> + + <property> + <name>yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds</name> + <value>3600</value> + </property> + + <property> + <name>yarn.timeline-service.entity-group-fs-store.done-dir</name> + <value>/ats/done/</value> + </property> + + <property> + <name>yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes</name> + <value>org.apache.tez.dag.history.logging.ats.TimelineCachePluginImpl</value> + </property> + + <property> + <name>yarn.timeline-service.entity-group-fs-store.retain-seconds</name> + <value>604800</value> + </property> + + <property> + <name>yarn.timeline-service.entity-group-fs-store.scan-interval-seconds</name> + <value>60</value> + </property> + + <property> + <name>yarn.timeline-service.entity-group-fs-store.summary-store</name> + <value>org.apache.hadoop.yarn.server.timeline.RollingLevelDBTimelineStore</value> + </property> + + <property> <name>yarn.timeline-service.generic-application-history.store-class</name> <value>org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore</value> </property> @@ -482,6 +544,11 @@ </property> <property> + <name>yarn.timeline-service.leveldb-state-store.path</name> + <value>/hadoop/yarn/timeline</value> + </property> + + <property> <name>yarn.timeline-service.leveldb-timeline-store.path</name> <value>/hadoop/yarn/timeline</value> </property> @@ -507,8 +574,23 @@ </property> <property> + <name>yarn.timeline-service.plugin.enabled</name> + <value>true</value> + </property> + + <property> + <name>yarn.timeline-service.recovery.enabled</name> + <value>true</value> + </property> + + <property> + <name>yarn.timeline-service.state-store-class</name> + <value>org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore</value> + </property> + + <property> <name>yarn.timeline-service.store-class</name> - <value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value> + <value>org.apache.hadoop.yarn.server.timeline.EntityGroupFSTimelineStore</value> </property> <property> @@ -522,6 +604,11 @@ </property> <property> + <name>yarn.timeline-service.version</name> + <value>1.5</value> + </property> + + <property> <name>yarn.timeline-service.webapp.address</name> <value>sandbox.hortonworks.com:8188</value> </property> @@ -531,4 +618,4 @@ <value>sandbox.hortonworks.com:8190</value> </property> -</configuration> +</configuration> \ No newline at end of file
