On September 25, 2002 10:56 am, Jason Hildebrand wrote:
> On Wed, 2002-09-25 at 08:35, Max Ischenko wrote:
> > Could someone point me to the webware code
> > which handles automagic reloading of servlet code?
>
> Reloading servlets is a nontrivial problem, since one must also
> check all modules imported from that servlet. AFAIK, no one has
> implemented a full-scale solution to this problem.
>
> For alternatives, see
> http://webware.colorstudy.net/twiki/bin/view/Webware/OneShot
Actually, I have implemented this in expWebware and it should be possible for
someone to implement in Webware. Unfortunately, I don't have time to do it
myself.
It works like this:
* the class that controls everything (AppServer in Webware's case, MultiServer
in expWebware) spawns an extra thread that maintains a list of all module
files and their mtimes. It sits in a loop and each cycle it checks to see if
any of the module files have been modified. If it finds any modifications it
sets the flag self._shouldRestart.
* the main threadLoop (also managed by AppServer in Webware) keeps an eye on
the self._shouldRestart flag and calls self._restart() if it is set to true.
NB: this method may only be called by the main thread.
* self._restart() shuts down the AppServer and all applications, flushe stdout
and stderr, then uses os.execve to restart the WebKit process with the same
arguments it was started with. The PID remains the same.
This approach is significantly simpler and faster than oneshot and less prone
to strange errors as the AppServer operates normally unless a module has
changed.
Here are some of difficulties/differences that might be encountered in
implementing this in Webware:
* WebKit.ThreadedAppServer.py is not cleanly object-oriented as several
critical parts of this module are handled via module-level functions (run(),
main(), and shutDown()) and global variables rather than being handled by the
ThreadedAppServer class. The code I've attached belongs to a single class in
expWebware, but in Webware it might need to be spread between
ThreadedAppServer and the module-level functions.
* The startup sequence for Webware is significantly different to that in
expWebware so I'm not certain that this os.execve approach will work.
Cheers,
Tavis
-------------------
def restart(self):
"""Tell the main thread to restart the server."""
self._shouldRestart = True
def _restart(self):
"""Restart the server by completely reinitializing the process with
os.execve(). This method can only be called by the main thread. If a
worker thread calls it, the process will freeze up."""
self.shutdown()
sys.stdout.flush()
sys.stderr.flush()
L = [sys.executable]
# shutdown has already completed so 'start' not 'restart'
L.extend([i.replace('restart','start') for i in sys.argv])
os.execve(L[0], L, os.environ)
def mainThreadWaitLoop(self):
"""The main thread waits here while the server is running and the
worker
threads are doing their thing. This function is called at the end of
the start sequence."""
while self.running:
time.sleep(0.2) #@@TR: softcode this later
if self._shouldRestart:
self._restart()
def activateAutoReload(self):
self._autoReload = True
## @@ Temporary
self._fileMonitorThread = t =
Thread(target=self._fileMonitorThreadLoop)
t.start()
def deactivateAutoReload(self):
self._autoReload = False
try:
self._fileMonitorThread.join()
except:
pass
def _fileMonitorThreadLoop(self,
getmtime=os.path.getmtime):
monitoredFiles = self._monitoredFiles
monitoredModules = self._monitoredModules
while self._autoReload:
time.sleep(1) #@@TR: softcode this later
for mod in sys.modules.values():
if not mod or mod in monitoredModules:
continue
monitoredModules.append(mod)
f2 = getattr(mod, '__orig_file__', False) # @@TR might rename
__orig_file__, this is used for cheetah and psp mods
f = getattr(mod, '__file__', False)
if f2 and f2 not in monitoredFiles.keys():
try:
monitoredFiles[f2] = getmtime(f2)
except OSError:
pass
elif f and f not in monitoredFiles.keys():
try:
monitoredFiles[f] = getmtime(f)
except OSError:
pass
for f, mtime in monitoredFiles.items():
try:
if mtime < getmtime(f):
print '*** The file', f, 'has changed. The server is
restarting now.'
sys.stdout.flush()
sys.stderr.flush()
self._autoReload = False
return self.restart()
except OSError:
print '*** The file', f, 'is no longer accesible The
server is restarting now.'
sys.stdout.flush()
sys.stderr.flush()
self._autoReload = False
return self.restart()
#!/usr/bin/env python
"""Provides a multi-port socket server that dispatches connections on multiple
ports to the services that are bound to them.
Its main use is to dispatch connections from Webware adapters to the WebKit
AppServer.
"""
# $Id: MultiServer.py,v 1.5 2002/09/25 18:32:24 tavis Exp $
__author__ = 'Tavis Rudd <[EMAIL PROTECTED]>'
__revision__ = "$Revision: 1.5 $"[11:-2]
##################################################
## DEPENDENCIES ##
import sys # used for printing to sys.stdout
import os # used to check os.name and for creating
# pipes
import os.path
import signal # used to capture interrupts
import select # used for catching select.error exceptions, etc.
import socket # this should be obvious
from threading import Thread # used to create worker threads
# that process the connections
from string import join as strJoin, lower as strLower
from exceptions import KeyboardInterrupt
from time import time as currentTime
import time
import traceback
# intra-packages imports ...
import Webware.Exceptions
from Webware.SettingsManager import SettingsManager
from PortDispatcher import ThreadedDispatcher
from _properties import Version
import _properties
##################################################
## GLOBALS & CONSTANTS ##
True = (1==1)
False = (1==0)
##################################################
## CLASSES ##
class Error(Webware.Exceptions.Error):
pass
class MultiServer(SettingsManager):
"""A TCP/IP socket server that listens to a specified list of ports and
dispatches incoming connections to the services that have been bound to
each port.
Services are instances of a subclass of the Service class. They can be
simple classes that accept the connection and return a string, or they can
be a a full-blown Application Server class that dispatches the request to an
appropriate servlet and returns the servlet's output. This modularized
approach makes it very easy to create custom application or admin servers,
or to debug this MultiServer independently of service.
Services must be thread-safe if you use the ThreadedDispatcher!!
After initializing the MultiServer you must bind at least one service to a
port using the method MultiServer.bindService(service); where 'service' is
an instance of a Service subclass. Internally, each Service will be wrapped
in a _Dispatcher object that will handle communication with the services
main socket in collaboration with the MultiServer.
By default the MultiServer relies on no external configuration or
cache files. However, almost everything about MultiServer is
configurable via subclassing or at runtime using its
updateSettings(newSettings, merge=True) method, so it is very easy to
create an interface for Webware that parses settings out of config files
upon startup and uses these settings to initialize the MultiServer,
an AppServer service, and Applications inside the AppServer.
Also note that MS COM can be used with this framework if the 'EnableCOM'
setting is set to True. When enabled it does the same as Webware's COMKit.
"""
_shouldRestart = False
_autoReload = False
def __init__(self, **kw):
"""Setup the server, but don't start it yet."""
SettingsManager.__init__(self)
self._initializeSettings()
if kw.has_key('settings'):
self.updateSettings(kw['settings'])
if self.setting('EnableCOM',False):
sys.coinit_flags = 0
self._addrToDispatcherMap = {}
self._serviceIDToAddrMap = {}
self._initFuncs = []
self._cleanupFuncs = []
self.running = False
## AutoReload stuff
# @@TR is this the right place for these
self._monitoredFiles = {}
self._monitoredModules = []
def _initializeSettings(self):
self._settings = {
'ServerName':"WebKit MultiServer",
'ServerVersion':Version,
'EventCheckInterval':100,
'EnableCOM':False,
'RecordPID':True,
'PIDFile':'webkit.pid',
'RecordRunSettings':True,
'RunSettingsFile':'webkit_run_settings.txt', # in the current working dir
'StartupNotice':'',
'ConfigFile':'',
'AutoReload':False,
}
def bindService(self, service, manageService=True):
"""Set the service object that will process connections on a given
hostName/port. 'address' should be a tuple in the form (hostName,
port). Note that hostName can be any legal name for the machine this
process is running on.
The 'manageService' arg specifies where this service needs to be
started and stopped by the MultiServer. The default is True,
but if you are binding a single service to multiple addresses you
should set this to True for the first port you address you bind to and
False for every address thereafter.
If returns a reference to the portDispatcher created for the service.
This reference is intended for internal use only!"""
if self._serviceIDToAddrMap.has_key(service.serviceID()):
raise Error(
'The MultiServer already has a service with the ID ' +
service.serviceID())
dispatcherClass = service.setting('DispatcherClass')
dispatcher = dispatcherClass(self, service, manageService=manageService)
hostName, port = service.serviceAddress()
addr = (socket.gethostbyname(hostName), port)
self._addrToDispatcherMap[addr] = dispatcher
self._serviceIDToAddrMap[service.serviceID()] = addr
if self.running:
dispatcher.start()
return dispatcher
def removeService(self, serviceID):
"""Shutdown and remove a service bound to a port."""
addr = self._serviceIDToAddrMap[serviceID]
self._addrToDispatcherMap[ addr ].shutdown()
del self._serviceIDToAddrMap[serviceID]
self._addrToDispatcherMap[ addr ] = None
del self._addrToDispatcherMap[ addr ]
def serviceIDToAddrMap(self):
return self._serviceIDToAddrMap
def connCountForService(self, serviceID):
"""Return the current connection count for service."""
address = self._serviceIDToAddrMap[serviceID]
return self._addrToDispatcherMap[ address ]._connCount
def startupNotice(self):
return self.setting('StartupNotice')
def activateAutoReload(self):
self._autoReload = True
## @@ Temporary
self._fileMonitorThread = t = Thread(target=self._fileMonitorThreadLoop)
t.start()
def deactivateAutoReload(self):
self._autoReload = False
try:
self._fileMonitorThread.join()
except:
pass
def start(self):
"""Start the portDispatchers then start listening on the active
ports."""
try:
self._startTime = currentTime()
## catch system interrupt signals ##
signal.signal(signal.SIGINT, self.shutdown)
signal.signal(signal.SIGTERM, self.shutdown)
self._PID = os.getpid()
## set the thread event check interval ##
sys.setcheckinterval( self.setting('EventCheckInterval') )
## call the custom init funcs
for func in self._initFuncs:
func()
print
print self.startupNotice()
print "="*80
print 'Starting', self.setting('ServerName'), 'at', \
time.asctime(time.localtime(time.time())), 'local time.'
self.printRunSettingsBlurb()
if not self._addrToDispatcherMap:
raise Error(self.setting('ServerName') +
' must have at least on Service to start.')
try:
self.running = True
## hook for storing info on the PID and ports being monitored ##
self.recordRunSettings()
print
for addr, dispatcher in self._addrToDispatcherMap.items():
dispatcher.start(indent=' ')
# note that a _NonThreadedDispatcher will grab control here
sys.stdout.flush()
if self.running:
# check because of _NonThreadedDispatcher's funny shutdown seq.
print
print self.setting('ServerName'), 'has been started'
print "="*80
print
sys.stdout.flush()
self.mainThreadWaitLoop()
except socket.error, e:
if e[0] == 98:
err = sys.stderr
traceback.print_exc(file=sys.stderr)
print >> err
print >> err, "************************* ERROR *************************"
print >> err, "*****"
print >> err, "***** CAN'T START:"
print >> err, "***** One of the network ports we need is still in use."
print >> err, "***** Try again in 15 seconds. "
print >> err, "***** If it still doesn't work, there might be another" \
" process using the port."
print >> err, "*****"
print >> err
err.flush()
self.shutdown()
return
except KeyboardInterrupt:
err = sys.stderr
print
print >> err, "************************* NOTICE *************************"
print >> err, "*****"
print >> err, "***** A Keyboard Interrupt (i.e. CTRL-C) was received."
print >> err, "***** Shutting down now!"
print >> err, "*****"
print
err.flush()
self.shutdown()
return
except:
err = sys.stderr
traceback.print_exc(file=sys.stderr)
print >> err
print >> err, "************************* ERROR *************************"
print >> err, "*****"
print >> err, "***** Attempt to start", self.setting('ServerName'), "failed"
print >> err, "*****"
print >> err
err.flush()
self.shutdown()
def shutdown(self, signalNum=None, currentStackFrame=None):
"""Cleanup and shutdown the dispatcher."""
self.running = False
try: ## @@ Temporary
self.deactivateAutoReload()
except:
pass
print
print "="*80
print 'Shutting down', self.setting('ServerName'), 'at', \
time.asctime(time.localtime(time.time())), 'local time.'
self.printRunSettingsBlurb()
print
## shutdown the portDispatchers
for addr, portDispatcher in self._addrToDispatcherMap.items():
portDispatcher.shutdown(indent=' ')
self._addrToDispatcherMap[addr] = None
del self._addrToDispatcherMap[addr]
## call the custom cleanupFuncs
for func in self._cleanupFuncs:
func()
print
print self.setting('ServerName'), 'has been shutdown'
print "="*80
if self.setting('RecordPID'):
try:
os.remove(self.setting('PIDFile'))
print "Temporary file", self.setting('PIDFile'), "has been removed."
except:
print "Couldn't remove the PIDFile (" + self.setting('PIDFile') + ")."
if self.setting('RecordRunSettings'):
try:
os.remove(self.setting('RunSettingsFile'))
print "Temporary file", self.setting('RunSettingsFile'), \
"has been removed."
except:
print ("Couldn't remove the runSettingsFile (" +
self.setting('RunSettingsFile') + ").")
print
def mainThreadWaitLoop(self):
"""The main thread waits here while the server is running and the worker
threads are doing their thing. This function is called at the end of
the start sequence."""
while self.running:
time.sleep(0.2) #@@TR: softcode this later
if self._shouldRestart:
self._restart()
def _fileMonitorThreadLoop(self,
getmtime=os.path.getmtime):
monitoredFiles = self._monitoredFiles
monitoredModules = self._monitoredModules
while self._autoReload:
time.sleep(1) #@@TR: softcode this later
for mod in sys.modules.values():
if not mod or mod in monitoredModules:
continue
monitoredModules.append(mod)
f2 = getattr(mod, '__orig_file__', False) # @@TR might rename __orig_file__
f = getattr(mod, '__file__', False)
if f2 and f2 not in monitoredFiles.keys():
try:
monitoredFiles[f2] = getmtime(f2)
except OSError:
pass
elif f and f not in monitoredFiles.keys():
try:
monitoredFiles[f] = getmtime(f)
except OSError:
pass
for f, mtime in monitoredFiles.items():
try:
if mtime < getmtime(f):
print '*** The file', f, 'has changed. The server is restarting now.'
sys.stdout.flush()
sys.stderr.flush()
self._autoReload = False
return self.restart()
except OSError:
print '*** The file', f, 'is no longer accesible The server is restarting now.'
sys.stdout.flush()
sys.stderr.flush()
self._autoReload = False
return self.restart()
def restart(self):
"""Tell the main thread to restart the server."""
self._shouldRestart = True
def _restart(self):
"""Restart the server by completely reinitializing the process with
os.execve(). This method can only be called by the main thread. If a
worker thread calls it, the process will freeze up."""
self.shutdown()
sys.stdout.flush()
sys.stderr.flush()
L = [sys.executable]
# shutdown has already completed so 'start' not 'restart'
L.extend([i.replace('restart','start') for i in sys.argv])
os.execve(L[0], L, os.environ)
def getUserInfo(self):
"""Returns the loginName, uid, and gid of the user running WebKit
"""
if os.name == 'posix':
loginName = os.environ['USER']
uid = os.getuid()
gid = os.getgid()
else:
try:
import win32api
loginName = win32api.GetUserName()
uid = 'N/A'
gid = 'N/A'
except:
loginName = 'N/A'
uid = 'N/A'
gid = 'N/A'
return loginName, uid, gid
def printRunSettingsBlurb(self):
loginName, uid, gid = self.getUserInfo()
serverName = self.setting('ServerName')
print ' Version:', Version
print ' Installed in:', os.path.split(_properties.__file__)[0]
if self.setting('ConfigFile'):
print ' Config file:', self.setting('ConfigFile')
print ' Working directory:', str(os.getcwd())
print ' Process ID: ', str(self._PID)
print ' Current user:', loginName, '(uid: '+ str(uid) + ', gid: ' + str(gid) + ')'
print ' Start time (local):', time.asctime(time.localtime(self._startTime))
print ' Start time (UTC):', time.asctime(time.gmtime(self._startTime))
def recordRunSettings(self):
"""A hook for storing a reference to the PID and server settings"""
loginName, uid, gid = self.getUserInfo()
if self.setting('RecordPID'):
PIDFile = self.setting('PIDFile')
PIDFile = self.settings()['PIDFile'] = os.path.abspath(PIDFile)
print ' Process ID recorded in: ', PIDFile
fp = open( self.setting('PIDFile'), 'w' )
fp.write(str(self._PID))
fp.close()
if self.setting('RecordRunSettings'):
# the runSettings file is written so that it is parsable by the
# ConfigParser module. It might make sense to just record this
# information in plain english and then record it again in a pickled
# dictionary that is saved alongside it. The pickled dict could
# also contain a copy of the config_file settings used to initialize
# WebKit.
runSettingsFile = self.setting('RunSettingsFile')
runSettingsFile = self.settings()['RunSettingsFile'] = os.path.abspath(runSettingsFile)
print ' Run settings recorded in: ', runSettingsFile
fp = open( self.setting('RunSettingsFile'), 'w' )
print >> fp, '[' + self.setting('ServerName') + ']'
print >> fp, 'start_time_utc:', time.asctime(time.gmtime(self._startTime))
print >> fp, 'start_time_local:', time.asctime(time.localtime(self._startTime))
print >> fp, 'pid:', str(self._PID)
if self.setting('ConfigFile'):
print >> fp, 'config_file:', self.setting('ConfigFile')
print >> fp, 'user_name:', loginName
print >> fp, 'user_id:', str(uid)
print >> fp, 'group_id:', str(gid)
print >> fp, 'current_working_dir:', os.getcwd()
print >> fp, 'server_version:', self.setting('ServerVersion')
print >> fp, 'python_version:', sys.version.replace('\n',' ')
print >> fp, 'python_path:', ';'.join(sys.path)
print >> fp, 'sys_event_check_interval:', self.setting('EventCheckInterval')
print >> fp, 'com_enabled:', self.setting('EnableCOM')
print >> fp
for serviceID, address in self._serviceIDToAddrMap.items():
dispatcher = self._addrToDispatcherMap[address]
print >> fp, '[' + serviceID + ']'
print >> fp, "address: %s, %s"% (address[0], address[1])
print >> fp, "threads:", dispatcher.setting('Threads')
print >> fp, "listen_queue_limit:", dispatcher.setting('ListenQueueLimit')
print >> fp
fp.close()
class Service(SettingsManager):
"""An abstract base class for MultiServer services.
Each instances settings must define ServiceID, ServiceName, HostName, Port,
DispatcherClass, and DispatcherSettings
"""
def _initializeSettings(self):
SettingsManager._initializeSettings(self)
self.updateSettings({'DispatcherSettings':{},
'DispatcherClass':ThreadedDispatcher,
'HostName':'localhost',
})
def start(self, indent=' '*2):
pass
def shutdown(self, indent=' '*2):
pass
def handleConnection(self, conn):
raise Webware.Exceptions.SubclassResponsibilityError
def serviceID(self):
return self.setting('ServiceID')
def serviceName(self):
return self.setting('ServiceName')
def serviceAddress(self):
return (self.setting('HostName'), self.setting('Port'))
##################################################
## Example Services ##
class HttpHelloWorld(Service):
def handleConnection(conn):
conn.send(
"HTTP/1.0 200 OK\r\nContent-type: text/html\r\n\r\nHello World!<BR><BR>\r\n")
conn.close()
class HttpHelloWorldLong(Service):
def handleConnection(conn, buffer="Hi There<BR>\n"*1000):
conn.send(
"HTTP/1.0 200 OK\r\nContent-type: text/html\r\n\r\nHello World!<BR><BR>\r\n")
reslen = len(buffer)
sent = 0
while sent < reslen:
try:
sent = sent + conn.send( buffer[sent:sent+8192] )
except socket.error, e:
break
conn.close()
class WebwareTestService(Service):
def handleConnection(conn, strJoin=''.join, BUFSIZE=8*1024):
"""For use with a Webware adapter. This won't work by itself.
"""
## get the request dict from the webserver adapter ##
recv = conn.recv
data = []
while 1:
chunk = recv(BUFSIZE)
if not chunk:
break
data.append(chunk)
data = strJoin(data)
buffer = "Content-type: text/html\r\n\r\n\r\nThe webware " + \
"adapter sucessfully transmitted the request.<BR><BR>"
conn.shutdown(0)
conn.send(buffer)
conn.close()
class WebwareEchoService(Service):
def handleConnection(conn):
"""A trivial services that echoes the request data.
For use with a Webware adapter. This won't work by itself.
"""
## get the request dict from the webserver adapter ##
recv = conn.recv
BUFSIZE = 8*1024
data = []
#while 1:
chunk = recv(BUFSIZE)
#if not chunk:
# break
data.append(chunk)
data = strJoin(data, '')
conn.shutdown(0)
buffer = "Content-type: text/html\r\n\r\n" + data
## send the response buffer ##
sent = 0
reslen = len(buffer)
while sent < reslen:
try:
sent = sent + conn.send( buffer[sent:sent+8192] )
except socket.error, e:
break
conn.close()
##################################################
def main():
server = MultiServer()
server.bindService(
HttpHelloWorldLong('Hello World!', ('localhost', 8088),
settings={'Threads':15,
'SysCheckInterval':100,
},
)
)
server.start()
if __name__ == '__main__':
main()
