<configuration>
<property>
<name>tez.version</name>
<value>0.4.0</value>
</property>
<property>
<name>tez.lib.uris</name>
<value>${fs.default.name}/apps/tez-0.4.0-incubating-full,${
fs.default.name}/apps/tez-0.4.0-incubating-full/lib/</value>
</property>
<!-- Client Submission timeout value
when submitting DAGs to a session -->
<property>
<name>tez.session.client.timeout.secs</name>
<value>-1</value>
</property>
<!-- prewarm stuff -->
</configuration>
On Thu, May 15, 2014 at 12:11 PM, oracle cool <[email protected]> wrote:
> Hi
>
> We are running Hadoop 2.2 , Hive 0.13 - Hive Server1.
>
>
> When we enabled tez 0.4 ( tables are in rcfile, no vectorization) &
> executed the query, we see major improvement in query but when we run
> concurrent 10 session & execute same query ( select count(*) from tab ),
> One batch of 10 query executed but next batch hang after 3 query.
>
> -------------
> via hive server 1 jdbc connection - 10 connection which execute same query
> after 1 min.
>
> query execution time in first batch is 15 sec.
>
> set hive.execution.engine=tez;
> select count(*) from table1;
>
> ---------------
>
>
> yarn-site.xml
> -------------------
>
> <configuration>
> <!-- Site specific YARN configuration properties -->
> <property>
> <name>yarn.nodemanager.aux-services</name>
> <value>mapreduce_shuffle</value>
> </property>
> <property>
>
> <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
> <value>org.apache.hadoop.mapred.ShuffleHandler</value>
> </property>
> <property>
> <name>yarn.resourcemanager.resource-tracker.address</name>
> <value>proj233:8025</value>
> </property>
> <property>
> <name>yarn.resourcemanager.scheduler.address</name>
> <value>proj233:8030</value>
> </property>
> <property>
> <name>yarn.resourcemanager.address</name>
> <value>proj233:8040</value>
> </property>
> <property>
> <name>yarn.scheduler.minimum-allocation-mb</name>
> <value>1024</value>
> </property>
> <property>
> <name>yarn.scheduler.maximum-allocation-mb</name>
> <value>40960</value>
> </property>
> <property>
> <name>yarn.scheduler.minimum-allocation-vcores</name>
> <value>1</value>
> </property>
> <property>
> <name>yarn.scheduler.maximum-allocation-vcores</name>
> <value>24</value>
> </property>
> <property>
> <name>yarn.nodemanager.resource.memory-mb</name>
> <value>40960</value>
> </property>
> <property>
> <name>yarn.nodemanager.resource.cpu-vcores</name>
> <value>1</value>
> </property>
> <property>
> <name>yarn.resourcemanager.scheduler.class</name>
>
> <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
> </property>
> </configuration>
>
>
> Mapred-site.xml
> ---------------------
> <configuration>
>
> <property>
> <name>mapreduce.framework.name</name>
> <value>yarn-tez</value>
> </property>
>
> <property>
> <name>yarn.nodemanager.vmem-pmem-ratio</name>
> <value>2.1</value>
> </property>
> <property>
> <name>mapreduce.task.io.sort.mb</name>
> <value>32</value>
> </property>
> <property>
> <name>yarn.app.mapreduce.am.resource.mb</name>
> <value>1024</value>
> </property>
> <property>
> <name>yarn.app.mapreduce.am.command-opts</name>
> <value>-Xmx768m</value>
> </property>
> <property>
> <name>mapreduce.map.cpu.vcores</name>
> <value>1</value>
> </property>
> <property>
> <name>mapreduce.reduce.cpu.vcores</name>
> <value>1</value>
> </property>
> <property>
> <name>mapreduce.map.memory.mb</name>
> <value>1024</value>
> </property>
> <property>
> <name>mapreduce.map.java.opts</name>
> <value>-Xmx768m</value>
> </property>
> <property>
> <name>mapreduce.reduce.memory.mb</name>
> <value>1024</value>
> </property>
> <property>
> <name>mapreduce.reduce.java.opts</name>
> <value>-Xmx768m</value>
> </property>
> </configuration>
>
>
> hdfs-site.xml
> --------------------
> <configuration>
> <property>
> <name>dfs.replication</name>
> <value>3</value>
> </property>
> <property>
> <name>dfs.namenode.name.dir</name>
> <value>file:/Data0/yarn_data/hdfs/namenode</value>
> </property>
> <property>
> <name>dfs.datanode.data.dir</name>
>
> <value>/Data0/yarn_data/hdfs/datanode,/Data1/yarn_data/hdfs/datanode,/Data3/yarn_data/hdfs/datanode,/Data4/yarn_data/hdfs/datanode</value>
> </property>
> <property>
> <name>dfs.heartbeat.interval</name>
> <value>3</value>
> </description>
> </property>
>
> <property>
> <name>dfs.safemode.threshold.pct</name>
> <value>1.0f</value>
> </property>
> <property>
> <name>dfs.block.size</name>
> <value>134217728</value>
> </property>
> <property>
> <name>dfs.datanode.max.xcievers</name>
> <value>16384</value>
> </property>
>
> <property>
> <name>dfs.client.block.write.replace-datanode-on-failure.enable</name>
> <value>false</value>
> </property>
> <property>
> <name>dfs.datanode.handler.count</name>
> <value>10</value>
> </property>
> <property>
> <name>dfs.datanode.socket.write.timeout</name>
> <value>0</value>
> </configuration>
>
>
>
> Thanks
> SS
>
>
