Hi All,
I have a problem which is that memory usage of twisted is increasing but not
decreasing over time.
After few days, it reaches 520MB. You may find the codes below;
How may I reduce the usage of memory?
Twisted version: 8.2.0
Python version: 2.4.3
[r...@labris init.d]# cat /usr/bin/labris-flyng.tac
#!/usr/bin/python
import os
from twisted.application import internet, service
from twisted.internet import reactor
from twisted.enterprise import adbapi
from twisted.plugin import getPlugins
from twisted.python import log
import labris.flyng.iflyng as iflyng
import labris.flyng.config as config
import labris.flyng.plugins as pplugins
import labris.flyng.protocols as flyng_protocols
from labris.flyng.config import flyng_config
# Database type to package dictionary
DB_PACKAGE_DICT = {
"postgresql" : "psycopg2"
}
def initDatabase(config):
"""
Create database connection pool
"""
dbtype = config.getDBType()
dbpackage = DB_PACKAGE_DICT.get(dbtype, None)
if not dbpackage:
print "Database type % is not supported yet!" %(dbtype)
sys.exit(1)
if dbtype == "mysql":
dbpool = adbapi.ConnectionPool(
dbpackage,
config.getDBHost(),
config.getDBUser(),
config.getDBPassword(),
config.getDBName()
)
else:
dbpool = adbapi.ConnectionPool(
dbpackage,
database=config.getDBName(),
user=config.getDBUser(),
password=config.getDBPassword(),
host=config.getDBHost()
)
dbpool.connect()
return dbpool
def setupPlugins(config, dbpool):
"""
Initialize and setup plugins
"""
for plugin in getPlugins(iflyng.IFlyngPlugin, pplugins):
print "* Plugin %s" %(plugin.name)
if not config.isEnabled(plugin.name):
print " - Not enabled"
continue
plugin.init(config, dbpool)
for type, port in config.getListeningPorts(plugin.name):
port = port.replace('"', "")
if type == "tcp":
print " -> TCP port: %s" %(port)
factory = flyng_protocols.FlyngServerFactory(plugin)
internet.TCPServer(
int(port),
factory
).setServiceParent(application)
elif type == "udp":
print " -> UDP port: %s" %(port)
internet.UDPServer(
int(port),
flyng_protocols.FlyngUDPProtocol(plugin)
).setServiceParent(application)
elif type == "unix":
print " -> UNIX stream: %s" %(port)
if os.path.exists(port):
os.unlink(port)
factory = flyng_protocols.FlyngServerFactory(plugin)
internet.UNIXServer(
port,
factory
).setServiceParent(application)
else:
raise "connection type %s is not supported" %(port)
#log.startLogging(open("/var/log/labris-flyng.log", "a"))
config = config.Configuration()
uid, gid = config.getRunAs()
application = service.Application('labris-flyng', uid, gid)
dbpool = initDatabase(config)
setupPlugins(config, dbpool)
-------------------------------------------------------------------
[r...@labris init.d]# cat /usr/bin/labris-pyng.tac
#!/usr/bin/python
"""
A syslog-ng to database mapping service
"""
import os
from twisted.internet import protocol, reactor, defer
from twisted.application import internet, service
from twisted.python import log
from labris.logutils import factories
from labris.logutils.config import pyng_config
class LogFactory(protocol.ServerFactory):
pass
class UnixLogFactory(protocol.ServerFactory):
pass
def openUNIXStreams(configuration, application):
for logtype, stream in configuration.getUnixStreams():
try:
if not pyng_config.getboolean(logtype.strip(), "enabled"):
print "Warning: UNIX stream: ", logtype, " is DISABLED"
continue
print "\nPlugin: %s [UNIX-Stream(%s)]" %(logtype, stream)
factory = UnixLogFactory()
factory.protocol =
factories.ProtocolFactory.getLogProtocol(logtype.strip(), "stream")
# delete old stream
if os.path.exists(stream):
os.unlink(stream)
internet.UNIXServer(stream.strip(),
factory).setServiceParent(application)
except factories.NoSuchProtocolException, e:
print "Error: no such protocol:", e.getProtoname()
for logtype, sock in configuration.getUnixDgrams():
try:
if not pyng_config.getboolean(logtype.strip(), "enabled"):
print "Warning: UNIX dgram: ", logtype, " is DISABLED"
continue
print "\nPlugin: %s [UNIX-Datagram(%s)]" %(logtype, stream)
protocol =
factories.ProtocolFactory.getLogProtocol(logtype.strip(),
"datagram")
# delete old stream
if os.path.exists(sock):
os.unlink(sock)
internet.UNIXDatagramServer(sock.strip(),
protocol()).setServiceParent(application)
except factories.NoSuchProtocolException, e:
print "Error: no such protocol:", e.getProtoname()
def openSockets(configuration, application):
for logtype, hostname, port in configuration.getTCP():
try:
if not pyng_config.getboolean(logtype.strip(), "enabled"):
print "Warning: TCP socket: ", logtype, " is DISABLED"
continue
print "\nPlugin: %s [TCP => port:%d]" %(logtype, int(port))
factory = LogFactory()
factory.protocol =
factories.ProtocolFactory.getLogProtocol(logtype.strip(), "stream")
internet.TCPServer(int(port), factory).setServiceParent(application)
except factories.NoSuchProtocolException, e:
print "Error: no such protocol:", e.getProtoname()
for logtype, hostname, port in configuration.getUDP():
try:
if not pyng_config.getboolean(logtype.strip(), "enabled"):
print "Warning: UDP socket: ", logtype, " is DISABLED"
continue
print "\nPlugin: %s [UDP => port:%d]" %(logtype, int(port))
protocol =
factories.ProtocolFactory.getLogProtocol(logtype.strip(),
"datagram")
internet.UDPServer(int(port),
protocol()).setServiceParent(application)
except factories.NoSuchProtocolException, e:
print "Error: no such protocol:", e.getProtoname()
#log.startLogging(open("/var/log/labris-pyng.log", "a"))
from labris.logutils.dao.pyngdb import PyNgDB
PyNgDB.initDatabase(pyng_config.bind())
application = service.Application('labris-pyng', pyng_config.uid(),
pyng_config.gid())
openUNIXStreams(pyng_config, application)
openSockets(pyng_config, application)
_______________________________________________
Twisted-Python mailing list
Twisted-Python@twistedmatrix.com
http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-python
