I've been messing with a patch to the depot server which is helping
me to suss out various transport issues. I've dubbed it "nasty mode".
To use:
./depot.py --nasty=<nasty-level> -d /aux0/test/repo -p 22222
The nasty-level is a number between 1 and 100 (0 would be "off")
which sets the likelihood of the server doing something nasty
like:
- Timing out
- Sending a bad response
- Sending no response
- Sending a partial response
- Todo: Sending a corrupted response
- Todo: Sending the wrong manifest
- Todo: Sending some random HTML instead of e.g. the catalog
Running the client against a server configured this way yields
a variety of tracebacks which hopefully we can clean up.
Attached is the patch. I'm leaving for vacation soon-- hopefully
someone can take this over and get it pushed. I think it will make
resiliency testing a lot more tractable.
For a reasonable test, I would start with nasty=5.
I'm not sure this patch is 100% right, FYI-- when I try to return
a '504' to the client, it's seeing it as 500. So perhaps some
cherrypy hackery is needed.
Another possibility would be to clean this up to allow the user
to toggle on and off different pathologies...
--nasty=504,timeout,multipletimeout --nasty-level=5
Thoughts?
-dp
--
Daniel Price - Solaris Kernel Engineering - [EMAIL PROTECTED] - blogs.sun.com/dp
diff -r 48f7e9d2b1ac src/depot.py
--- a/src/depot.py Tue Oct 07 15:41:50 2008 -0500
+++ b/src/depot.py Tue Oct 07 17:43:46 2008 -0700
@@ -65,6 +65,8 @@
REINDEX_DEFAULT = False
# Not in mirror mode by default
MIRROR_DEFAULT = False
+# Not in nasty mode by default
+NASTY_DEFAULT = 0
import getopt
import logging
@@ -109,15 +111,18 @@
emsg(text)
print """\
-Usage: /usr/lib/pkg.depotd [--readonly] [--rebuild] [--mirror] [--proxy-base
url]
- [--log-access dest] [--log-errors dest] [-d repo_dir] [-p port]
- [-s threads] [-t socket_timeout]
+Usage: /usr/lib/pkg.depotd [--readonly] [--rebuild] [--mirror] [--nasty]
+ [--proxy-base url] [--log-access dest] [--log-errors dest]
+ [-d repo_dir] [-p port] [-s threads] [-t socket_timeout]
--readonly Read-only operation; modifying operations disallowed
--rebuild Re-build the catalog from pkgs in depot
Cannot be used with --readonly or --mirror
--mirror Content mirror mode. Publishing and metadata operations
disabled
+ --nasty=level "Be a bad server" mode. Time out, fail, restart, etc.
+ at random. Level of nastiness (1 to 100) can be
+ configured
--proxy-base The url to use as the base for generating internal
redirects and content.
--log-access The destination for any access related information
@@ -148,6 +153,7 @@
reindex = REINDEX_DEFAULT
proxy_base = None
mirror = MIRROR_DEFAULT
+ nasty = NASTY_DEFAULT
if "PKG_REPO" in os.environ:
repo_path = os.environ["PKG_REPO"]
@@ -170,7 +176,7 @@
opt = None
try:
long_opts = ["readonly", "rebuild", "mirror", "refresh-index",
- "proxy-base="]
+ "proxy-base=", "nasty="]
for opt in log_opts:
long_opts.append("%s=" % opt.lstrip('--'))
opts, pargs = getopt.getopt(sys.argv[1:], "d:np:s:t:",
@@ -230,6 +236,9 @@
elif opt == "--mirror":
mirror = True
+ elif opt == "--nasty":
+ nasty = int(arg)
+
except getopt.GetoptError, e:
usage("pkg.depotd: %s" % e.msg)
except OptionError, e:
@@ -270,6 +279,9 @@
if mirror:
scfg.set_mirror()
+
+ if nasty:
+ scfg.set_nasty(nasty)
try:
scfg.init_dirs()
diff -r 48f7e9d2b1ac src/modules/server/config.py
--- a/src/modules/server/config.py Tue Oct 07 15:41:50 2008 -0500
+++ b/src/modules/server/config.py Tue Oct 07 17:43:46 2008 -0700
@@ -30,6 +30,7 @@
import os.path
import statvfs
import shutil
+import random
import pkg.server.catalog as catalog
import pkg.updatelog as updatelog
@@ -64,6 +65,7 @@
self.flist_requests = 0
self.flist_files = 0
self.pkgs_renamed = 0
+ self.nasty = 0
def init_dirs(self):
@@ -109,6 +111,25 @@
def set_mirror(self):
self.mirror = True
+
+ def set_nasty(self, level):
+ self.nasty = level
+
+
+ def need_nasty(self):
+ if random.randint(1,100) <= self.nasty:
+ return True
+ return False
+
+ def need_nasty_rarely(self):
+ if random.randint(1,500) <= self.nasty:
+ return True
+ return False
+
+ def need_nasty_very_rarely(self):
+ if random.randint(1,2000) <= self.nasty:
+ return True
+ return False
def is_read_only(self):
return self.read_only
@@ -183,6 +204,9 @@
self.file_requests, self.flist_requests, self.flist_files,
self.pkgs_renamed)
+ if self.nasty:
+ ret += """<br/><b><font color="red">""" + \
+ """Warning: nasty mode enabled</font></b>"""
return ret
def inc_catalog(self):
diff -r 48f7e9d2b1ac src/modules/server/repository.py
--- a/src/modules/server/repository.py Tue Oct 07 15:41:50 2008 -0500
+++ b/src/modules/server/repository.py Tue Oct 07 17:43:46 2008 -0700
@@ -264,6 +264,15 @@
# have been sent. But because we've sent data already (never
# mind the response header), we can't raise an exception here,
# or an INTERNAL_SERVER_ERROR header will get sent as well.
+
+ # NASTY
+ if self.scfg.need_nasty():
+ return
+
+ # NASTY
+ if self.scfg.need_nasty():
+ raise cherrypy.HTTPError(httplib.GATEWAY_TIMEOUT)
+
try:
return self.scfg.updatelog.send(cherrypy.request,
cherrypy.response)
@@ -283,6 +292,16 @@
interpretation of the catalog and its image policies. """
self.scfg.inc_manifest()
+
+ # NASTY
+ # emit an http error code the client should
+ # know how to retry
+ if self.scfg.need_nasty():
+ raise cherrypy.HTTPError(httplib.GATEWAY_TIMEOUT)
+
+ if self.scfg.need_nasty_rarely():
+ import time
+ time.sleep(10)
# Parse request into FMRI component and decode.
try:
@@ -328,12 +347,27 @@
try:
self.scfg.inc_flist()
+ # NASTY
+ if self.scfg.need_nasty_rarely():
+ return
+
+ # NASTY
+ # emit an http error code the client should
+ # know how to retry
+ if self.scfg.need_nasty():
+ raise
cherrypy.HTTPError(httplib.GATEWAY_TIMEOUT)
+
# Create a dummy file object that hooks to the write()
# callable which is all tarfile needs to output the
# stream. This will write the bytes to the client
# through our parent server process.
f = Dummy()
f.write = cherrypy.response.write
+
+ # NASTY
+ if self.scfg.need_nasty():
+ f.write("NASTY")
+ return
tar_stream = tarfile.open(mode = "w|",
fileobj = f)
@@ -351,7 +385,24 @@
cherrypy.request.hooks.attach('on_end_request',
self._tar_stream_close, failsafe = True)
+ if self.scfg.need_nasty():
+ import time
+ time.sleep(10)
+
for v in params.values():
+ # NASTY
+ if self.scfg.need_nasty_rarely():
+ break
+
+ # NASTY
+ if self.scfg.need_nasty_rarely():
+ continue
+
+ # NASTY
+ if self.scfg.need_nasty_very_rarely():
+ import time
+ time.sleep(10)
+
filepath = os.path.normpath(os.path.join(
self.scfg.file_root,
misc.hash_file_name(v)))
_______________________________________________
pkg-discuss mailing list
[email protected]
http://mail.opensolaris.org/mailman/listinfo/pkg-discuss
