Hey folks,
We (Red Hat) get the occasional complaint, particularly from jboss
folks, that setting up large page access for java app servers is too
difficult. As a result, I was asked to throw together a scriptlet to
make life easier for such people.
The attached python script has been used successfully on Red Hat
Enterprise Linux 5, Fedora 11 and Fedora 12, and is likely to work for
other distros (though possibly with some minor tweaking required). I
believe we're intending to include it in an upcoming RHEL release, and
if at all possible and appropriate, I'd appreciate seeing it included in
the official libhugetlbfs distribution for the benefit of others (and
ultimately for RHEL too, so we don't have to carry anything
out-of-tree), even if it were just under a not-installed-by-default
contrib/ directory.
Comments welcomed... Even if they are "wow, this sucks horribly, please
go away" (but hopefully they aren't). :)
--
Jarod Wilson
ja...@redhat.com
#!/usr/bin/python
#
# Tool to set up Linux large page support with minimal effort
#
# by Jarod Wilson <ja...@redhat.com>
# (c) Red Hat, Inc., 2009
#
import os
debug = False
# config files we need access to
sysctlConf = "/etc/sysctl.conf"
if not os.access(sysctlConf, os.W_OK):
print "Cannot access %s" % sysctlConf
limitsConf = "/etc/security/limits.d/hugepages.conf"
if not os.access(limitsConf, os.W_OK):
print "Cannot access %s" % limitsConf
# Figure out what we've got in the way of memory
memInfo = open("/proc/meminfo").readlines()
memTotal = 0
hugePages = 0
hugePageSize = 0
for line in memInfo:
if line.startswith("MemTotal:"):
memTotal = int(line.split()[1])
break
for line in memInfo:
if line.startswith("HugePages_Total:"):
hugePages = int(line.split()[1])
break
for line in memInfo:
if line.startswith("Hugepagesize:"):
hugePageSize = int(line.split()[1])
break
# Get initial sysctl settings
shmmax = 0
nr_hugepages = 0
hugeGID = 0
sysctlCur = os.popen("/sbin/sysctl -a").readlines()
for line in sysctlCur:
if line.startswith("kernel.shmmax = "):
shmmax = int(line.split()[2])
break
for line in sysctlCur:
if line.startswith("vm.nr_hugepages = "):
nr_hugepages = int(line.split()[2])
break
for line in sysctlCur:
if line.startswith("vm.hugetlb_shm_group = "):
hugeGID = int(line.split()[2])
break
# translate group into textual version
hugeGIDName = "null"
groupNames = os.popen("/usr/bin/getent group").readlines()
for line in groupNames:
curGID = int(line.split(":")[2])
if curGID == hugeGID:
hugeGIDName = line.split(":")[0]
break
# dump system config as we see it before we start tweaking it
print "Current configuration:"
print " * Total System Memory......: %6d MB" % (memTotal / 1024)
print " * Shared Mem Max Mapping...: %6d MB" % (shmmax / (1024 * 1024))
print " * System Huge Page Size....: %6d MB" % (hugePageSize / 1024)
print " * Number of Huge Pages.....: %6d" % hugePages
print " * Total size of Huge Pages.: %6d MB" % (hugePages * hugePageSize / 1024)
print " * Remaining System Memory..: %6d MB" % ((memTotal / 1024) - (hugePages * hugePageSize / 1024))
print " * Huge Page User Group.....: %s (%d)" % (hugeGIDName, hugeGID)
print
# determine some sanity safeguards
halfOfMem = memTotal / 2
allMemLess2G = memTotal - 2048000
if halfOfMem >= allMemLess2G:
maxHugePageReqKB = halfOfMem
else:
maxHugePageReqKB = allMemLess2G
maxHugePageReqMB = maxHugePageReqKB / 1024
maxHugePageReq = maxHugePageReqKB / hugePageSize
# ask how memory they want to allocate for huge pages
userIn = None
while not userIn:
try:
userIn = raw_input("How much memory would you like to allocate for huge pages? "
"(input in MB, unless postfixed with GB): ")
if userIn[-2:] == "GB":
userHugePageReqMB = int(userIn[0:-2]) * 1024
elif userIn[-1:] == "G":
userHugePageReqMB = int(userIn[0:-1]) * 1024
elif userIn[-2:] == "MB":
userHugePageReqMB = int(userIn[0:-2])
elif userIn[-1:] == "MB":
userHugePageReqMB = int(userIn[0:-1])
else:
userHugePageReqMB = int(userIn)
if userHugePageReqMB > maxHugePageReqMB:
userIn = None
print "Sorry, the most I'll let you allocate is %d MB, try again!" % maxHugePageReqMB
else:
break
except ValueError:
userIn = None
print "Input must be an integer, please try again!"
userHugePageReqKB = userHugePageReqMB * 1024
userHugePagesReq = userHugePageReqKB / hugePageSize
print "Okay, we'll try to allocate %d MB for huge pages..." % userHugePageReqMB
print
# some basic user input validation
badchars = list(' \\\'":;~`!$^&*(){}[]?/><,')
inputIsValid = False
foundbad = False
# ask for the name of the group allowed access to huge pages
while inputIsValid == False:
userGroupReq = raw_input("What group should have access to the huge pages? "
"(The group will be created, if need be): ")
for char in badchars:
if char in userGroupReq:
foundbad = True
print "Illegal characters in group name, please try again!"
break
if foundbad == False:
inputIsValid = True
print "Okay, we'll give group %s access to the huge pages" % userGroupReq
# see if group already exists, use it if it does, if not, create it
userGIDReq = -1
for line in groupNames:
curGroupName = line.split(":")[0]
if curGroupName == userGroupReq:
userGIDReq = int(line.split(":")[2])
break
if userGIDReq > -1:
print "Group %s (gid %d) already exists, we'll use it" % (userGroupReq, userGIDReq)
else:
if debug == False:
os.popen("/usr/sbin/groupadd %s" % userGroupReq)
else:
print "/usr/sbin/groupadd %s" % userGroupReq
groupNames = os.popen("/usr/bin/getent group %s" % userGroupReq).readlines()
for line in groupNames:
curGroupName = line.split(":")[0]
if curGroupName == userGroupReq:
userGIDReq = int(line.split(":")[2])
break
print "Created group %s (gid %d) for huge page use" % (userGroupReq, userGIDReq)
print
# basic user input validation, take 2
# space is valid in this case, wasn't in the prior incarnation
badchars = list('\\\'":;~`!$^&*(){}[]?/><,')
inputIsValid = False
# ask for user(s) that should be in the huge page access group
while inputIsValid == False:
userUsersReq = raw_input("What user(s) should have access to the huge pages (space-delimited list, users created as needed)? ")
for char in badchars:
if char in userUsersReq:
foundbad = True
print "Illegal characters in user name(s) or invalid list format, please try again!"
break
if foundbad == False:
inputIsValid = True
# see if user(s) already exist(s)
curUserList = os.popen("/usr/bin/getent passwd").readlines()
hugePageUserList = userUsersReq.split()
for hugeUser in hugePageUserList:
userExists = False
for line in curUserList:
curUser = line.split(":")[0]
if curUser == hugeUser:
print "Adding user %s to huge page group" % hugeUser
userExists = True
if debug == False:
os.popen("/usr/sbin/usermod -a -G %s %s" % (userGroupReq, hugeUser))
else:
print "/usr/sbin/usermod -a -G %s %s" % (userGroupReq, hugeUser)
if userExists == True:
break
if userExists == False:
print "Creating user %s with membership in huge page group" % hugeUser
if debug == False:
os.popen("/usr/sbin/useradd %s -G %s" % (hugeUser, userGroupReq))
else:
print "/usr/sbin/useradd %s -G %s" % (hugeUser, userGroupReq)
print
# set sysctl values for the current running environment
if debug == False:
os.popen("/sbin/sysctl -w kernel.shmmax=%d" % (memTotal * 1024))
os.popen("/sbin/sysctl -w vm.nr_hugepages=%d" % userHugePagesReq)
os.popen("/sbin/sysctl -w vm.hugetlb_shm_group=%d" % userGIDReq)
else:
print "/sbin/sysctl -w kernel.shmmax=%d" % (memTotal * 1024)
print "/sbin/sysctl -w vm.nr_hugepages=%d" % userHugePagesReq
print "/sbin/sysctl -w vm.hugetlb_shm_group=%d" % userGIDReq
print
# write out sysctl config changes to persist across reboot
if debug == False:
sysctlConfLines = "# sysctl configuration\n"
if os.access(sysctlConf, os.W_OK):
try:
sysctlConfLines = open(sysctlConf).readlines()
os.rename(sysctlConf, sysctlConf + ".backup")
print("Saved original %s as %s.backup" % (sysctlConf, sysctlConf))
except:
pass
fd = open(sysctlConf, "w")
for line in sysctlConfLines:
if line.startswith("kernel.shmmax"):
continue
elif line.startswith("vm.nr_hugepages"):
continue
elif line.startswith("vm.hugetlb_shm_group"):
continue
else:
fd.write(line);
fd.write("kernel.shmmax = %d\n" % (memTotal * 1024))
fd.write("vm.nr_hugepages = %d\n" % userHugePagesReq)
fd.write("vm.hugetlb_shm_group = %d\n" % userGIDReq)
fd.close()
else:
print "Add to %s:" % sysctlConf
print "kernel.shmmax = %d" % (memTotal * 1024)
print "vm.nr_hugepages = %d" % userHugePagesReq
print "vm.hugetlb_shm_group = %d" % userGIDReq
print
# write out limits.conf changes to persist across reboot
if debug == False:
limitsConfLines = "# Huge page access configuration\n"
if os.access(limitsConf, os.W_OK):
try:
limitsConfLines = open(limitsConf).readlines()
os.rename(limitsConf, limitsConf + ".backup")
print("Saved original %s as %s.backup" % (limitsConf, limitsConf))
except:
pass
fd = open(limitsConf, "w")
for line in limitsConfLines:
cfgExist = False
for hugeUser in hugePageUserList:
if line.split()[0] == hugeUser:
cfgExist = True
if cfgExist == True:
continue
else:
fd.write(line)
for hugeUser in hugePageUserList:
fd.write("%s soft memlock %d\n" % (hugeUser, userHugePageReqKB))
fd.write("%s hard memlock %d\n" % (hugeUser, userHugePageReqKB))
fd.close()
else:
print "Add to %s:" % limitsConf
for hugeUser in hugePageUserList:
print "%s soft memlock %d" % (hugeUser, userHugePageReqKB)
print "%s hard memlock %d" % (hugeUser, userHugePageReqKB)
# dump the final configuration of things now that we're done tweaking
print
print "Final configuration:"
print " * Total System Memory......: %6d MB" % (memTotal / 1024)
print " * Shared Mem Max Mapping...: %6d MB" % (memTotal / 1024)
print " * System Huge Page Size....: %6d MB" % (hugePageSize / 1024)
print " * Available Huge Pages.....: %6d" % userHugePagesReq
print " * Total size of Huge Pages.: %6d MB" % (userHugePagesReq * hugePageSize / 1024)
print " * Remaining System Memory..: %6d MB" % ((memTotal / 1024) - userHugePageReqMB)
print " * Huge Page User Group.....: %s (%d)" % (userGroupReq, userGIDReq)
print
------------------------------------------------------------------------------
Let Crystal Reports handle the reporting - Free Crystal Reports 2008 30-Day
trial. Simplify your report design, integration and deployment - and focus on
what you do best, core application coding. Discover what's new with
Crystal Reports now. http://p.sf.net/sfu/bobj-july
_______________________________________________
Libhugetlbfs-devel mailing list
Libhugetlbfs-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/libhugetlbfs-devel
