We use the hive thrift server and ruby client to submit queries to hive. But we
have noticed that the number of open file descriptors climbs steadily on the
machine running hive.
On a cluster of 21 nodes with hive running for around 5 days and processing
around 200 queries per day we see around 3,000 open file descriptors. This
growth continues until it saturates the system limits and then stops responding
to requests.
All of the connections appear to be to port 50010 (HDFS?) on the slave
machines.
Any help or suggestions would be greatly appreciated.
Thanks, Andy.
Below is an sample netstat output...
dep...@hadoopmaster:~$ ps aux|grep HiveServer|grep -v grep\ HiveServer|awk
'{print $2}'|while read i; do sudo netstat -np|grep \ "$i"\/java;done
tcp6 33380 0 192.168.31.200:41491 192.168.31.217:50010 ESTABLISHED
1768/java
tcp6 66800 0 192.168.31.200:39865 192.168.31.209:50010 ESTABLISHED
1768/java
tcp6 66800 0 192.168.31.200:33770 192.168.31.205:50010 ESTABLISHED
1768/java
tcp6 67040 0 192.168.31.200:45931 192.168.31.215:50010 ESTABLISHED
1768/java
tcp6 66880 0 192.168.31.200:55003 192.168.31.220:50010 ESTABLISHED
1768/java
tcp6 66760 0 192.168.31.200:52236 192.168.31.209:50010 ESTABLISHED
1768/java
tcp6 23033 0 192.168.31.200:51442 192.168.31.210:50010 ESTABLISHED
1768/java
tcp6 66504 0 192.168.31.200:34695 192.168.31.210:50010 ESTABLISHED
1768/java
tcp6 67040 0 192.168.31.200:45760 192.168.31.215:50010 ESTABLISHED
1768/java
tcp6 66680 0 192.168.31.200:55634 192.168.31.213:50010 ESTABLISHED
1768/java
tcp6 18536 0 192.168.31.200:47812 192.168.31.208:50010 ESTABLISHED
1768/java
tcp6 18536 0 192.168.31.200:55430 192.168.31.204:50010 ESTABLISHED
1768/java
tcp6 66680 0 192.168.31.200:33447 192.168.31.205:50010 ESTABLISHED
1768/java
tcp6 66680 0 192.168.31.200:44618 192.168.31.203:50010 ESTABLISHED
1768/java
tcp6 28269 0 192.168.31.200:60042 192.168.31.220:50010 ESTABLISHED
1768/java
tcp6 66560 0 192.168.31.200:60043 192.168.31.203:50010 ESTABLISHED
1768/java
tcp6 66760 0 192.168.31.200:38104 192.168.31.219:50010 ESTABLISHED
1768/java
tcp6 66760 0 192.168.31.200:36461 192.168.31.205:50010 ESTABLISHED
1768/java
tcp6 45333 0 192.168.31.200:43051 192.168.31.207:50010 ESTABLISHED
1768/java
tcp6 66680 0 192.168.31.200:32782 192.168.31.213:50010 ESTABLISHED
1768/java
tcp6 8794 0 192.168.31.200:33000 192.168.31.209:50010 ESTABLISHED
1768/java
tcp6 0 0 192.168.31.200:51397 192.168.31.200:50010 ESTABLISHED
1768/java
tcp6 66760 0 192.168.31.200:47584 192.168.31.206:50010 ESTABLISHED
1768/java
tcp6 66680 0 192.168.31.200:49115 192.168.31.219:50010 ESTABLISHED
1768/java
tcp6 66760 0 192.168.31.200:46942 192.168.31.207:50010 ESTABLISHED
1768/java
tcp6 66800 0 192.168.31.200:51521 192.168.31.212:50010 ESTABLISHED
1768/java
tcp6 66680 0 192.168.31.200:35897 192.168.31.205:50010 ESTABLISHED
1768/java
tcp6 42426 0 192.168.31.200:44042 192.168.31.208:50010 ESTABLISHED
1768/java
tcp6 37196 0 192.168.31.200:45567 192.168.31.212:50010 ESTABLISHED
1768/java
tcp6 66800 0 192.168.31.200:38376 192.168.31.207:50010 ESTABLISHED
1768/java
tcp6 66760 0 192.168.31.200:47279 192.168.31.203:50010 ESTABLISHED
1768/java
tcp6 66072 0 192.168.31.200:35648 192.168.31.217:50010 ESTABLISHED
1768/java
tcp6 66384 0 192.168.31.200:39264 192.168.31.206:50010 ESTABLISHED
1768/java
tcp6 8152 0 192.168.31.200:45371 192.168.31.209:50010 ESTABLISHED
1768/java
tcp6 66760 0 192.168.31.200:37021 192.168.31.209:50010 ESTABLISHED
1768/java
tcp6 34844 0 192.168.31.200:47316 192.168.31.219:50010 ESTABLISHED
1768/java
tcp6 66680 0 192.168.31.200:45311 192.168.31.215:50010 ESTABLISHED
1768/java
tcp6 66680 0 192.168.31.200:47921 192.168.31.219:50010 ESTABLISHED
1768/java
tcp6 66880 0 192.168.31.200:49766 192.168.31.220:50010 ESTABLISHED
1768/java
tcp6 47556 0 192.168.31.200:40457 192.168.31.208:50010 ESTABLISHED
1768/java
tcp6 67080 0 192.168.31.200:58490 192.168.31.220:50010 ESTABLISHED
1768/java
tcp6 66600 0 192.168.31.200:60526 192.168.31.203:50010 ESTABLISHED
1768/java
tcp6 66560 0 192.168.31.200:42714 192.168.31.220:50010 ESTABLISHED
1768/java
tcp6 66760 0 192.168.31.200:51175 192.168.31.201:50010 ESTABLISHED
1768/java
tcp6 66760 0 192.168.31.200:39403 192.168.31.206:50010 ESTABLISHED
1768/java
tcp6 66800 0 192.168.31.200:50538 192.168.31.206:50010 ESTABLISHED
1768/java
tcp6 66760 0 192.168.31.200:58341 192.168.31.219:50010 ESTABLISHED
1768/java
tcp6 66840 0 192.168.31.200:34861 192.168.31.204:50010 ESTABLISHED
1768/java
tcp6 18445 0 192.168.31.200:39932 192.168.31.202:50010 ESTABLISHED
1768/java
tcp6 20500 0 192.168.31.200:58142 192.168.31.206:50010 ESTABLISHED
1768/java
tcp6 66720 0 192.168.31.200:35214 192.168.31.218:50010 ESTABLISHED
1768/java
tcp6 66800 0 192.168.31.200:47101 192.168.31.215:50010 ESTABLISHED
1768/java
tcp6 15897 0 192.168.31.200:34267 192.168.31.202:50010 ESTABLISHED
1768/java
tcp6 66760 0 192.168.31.200:36748 192.168.31.218:50010 ESTABLISHED
1768/java
tcp6 66760 0 192.168.31.200:60095 192.168.31.203:50010 ESTABLISHED
1768/java
tcp6 66720 0 192.168.31.200:55681 192.168.31.213:50010 ESTABLISHED
1768/java
tcp6 66800 0 192.168.31.200:60046 192.168.31.214:50010 ESTABLISHED
1768/java
tcp6 22357 0 192.168.31.200:52681 192.168.31.206:50010 ESTABLISHED
1768/java
tcp6 66680 0 192.168.31.200:51024 192.168.31.204:50010 ESTABLISHED
1768/java
tcp6 66760 0 192.168.31.200:55210 192.168.31.215:50010 ESTABLISHED
1768/java
tcp6 66720 0 192.168.31.200:59062 192.168.31.220:50010 ESTABLISHED
1768/java
tcp6 66760 0 192.168.31.200:43004 192.168.31.214:50010 ESTABLISHED
1768/java
tcp6 72877 0 192.168.31.200:36495 192.168.31.200:50010 ESTABLISHED
1768/java
tcp6 66680 0 192.168.31.200:54799 192.168.31.211:50010 ESTABLISHED
1768/java
tcp6 5099 0 192.168.31.200:58771 192.168.31.214:50010 ESTABLISHED
1768/java
tcp6 66800 0 192.168.31.200:53092 192.168.31.210:50010 ESTABLISHED
1768/java
tcp6 65728 0 192.168.31.200:48137 192.168.31.215:50010 ESTABLISHED
1768/java
tcp6 66760 0 192.168.31.200:33296 192.168.31.208:50010 ESTABLISHED
1768/java
tcp6 66760 0 192.168.31.200:36596 192.168.31.211:50010 ESTABLISHED
1768/java
tcp6 66760 0 192.168.31.200:42156 192.168.31.215:50010 ESTABLISHED
1768/java
tcp6 66600 0 192.168.31.200:40412 192.168.31.220:50010 ESTABLISHED
1768/java
tcp6 66600 0 192.168.31.200:45285 192.168.31.212:50010 ESTABLISHED
1768/java
tcp6 66800 0 192.168.31.200:51538 192.168.31.213:50010 ESTABLISHED
1768/java
tcp6 66880 0 192.168.31.200:54502 192.168.31.203:50010 ESTABLISHED
1768/java
tcp6 28447 0 192.168.31.200:52610 192.168.31.203:50010 ESTABLISHED
1768/java
tcp6 9709 0 192.168.31.200:45242 192.168.31.219:50010 ESTABLISHED
1768/java
tcp6 66840 0 192.168.31.200:42858 192.168.31.212:50010 ESTABLISHED
1768/java
tcp6 66760 0 192.168.31.200:60246 192.168.31.218:50010 ESTABLISHED
1768/java
tcp6 28130 0 192.168.31.200:57436 192.168.31.208:50010 ESTABLISHED
1768/java
tcp6 66680 0 192.168.31.200:48177 192.168.31.206:50010 ESTABLISHED
1768/java
tcp6 66800 0 192.168.31.200:58655 192.168.31.219:50010 ESTABLISHED
1768/java
tcp6 0 0 192.168.31.200:42251 192.168.31.200:50010 ESTABLISHED
1768/java
tcp6 66800 0 192.168.31.200:44250 192.168.31.219:50010 ESTABLISHED
1768/java
tcp6 57047 0 192.168.31.200:56874 192.168.31.210:50010 ESTABLISHED
1768/java
tcp6 72857 0 192.168.31.200:45787 192.168.31.200:50010 ESTABLISHED
1768/java
tcp6 29953 0 192.168.31.200:46471 192.168.31.202:50010 ESTABLISHED
1768/java
tcp6 29953 0 192.168.31.200:60726 192.168.31.214:50010 ESTABLISHED
1768/java
tcp6 66760 0 192.168.31.200:60663 192.168.31.220:50010 ESTABLISHED
1768/java
tcp6 66880 0 192.168.31.200:47082 192.168.31.208:50010 ESTABLISHED
1768/java
tcp6 66760 0 192.168.31.200:60524 192.168.31.212:50010 ESTABLISHED
1768/java
tcp6 66880 0 192.168.31.200:53628 192.168.31.203:50010 ESTABLISHED
1768/java
tcp6 54203 0 192.168.31.200:37787 192.168.31.219:50010 ESTABLISHED
1768/java
tcp6 66640 0 192.168.31.200:47555 192.168.31.208:50010 ESTABLISHED
1768/java
tcp6 66800 0 192.168.31.200:59678 192.168.31.210:50010 ESTABLISHED
1768/java
tcp6 24663 0 192.168.31.200:45910 192.168.31.214:50010 ESTABLISHED
1768/java
tcp6 66720 0 192.168.31.200:35518 192.168.31.217:50010 ESTABLISHED
1768/java
tcp6 66608 0 192.168.31.200:40465 192.168.31.208:50010 ESTABLISHED
1768/java
tcp6 66760 0 192.168.31.200:36336 192.168.31.213:50010 ESTABLISHED
1768/java
tcp6 66800 0 192.168.31.200:45161 192.168.31.207:50010 ESTABLISHED
1768/java
tcp6 36029 0 192.168.31.200:40896 192.168.31.202:50010 ESTABLISHED
1768/java
tcp6 34261 0 192.168.31.200:57311 192.168.31.206:50010 ESTABLISHED
1768/java
tcp6 66720 0 192.168.31.200:42259 192.168.31.211:50010 ESTABLISHED
1768/java
tcp6 66880 0 192.168.31.200:40763 192.168.31.210:50010 ESTABLISHED
1768/java
.....
