Author: aum
Date: 2006-05-17 21:45:39 +0000 (Wed, 17 May 2006)
New Revision: 8752
Modified:
trunk/apps/pyFreenet/code.leo
trunk/apps/pyFreenet/fcp/node.py
trunk/apps/pyFreenet/fcp/sitemgr.py
trunk/apps/pyFreenet/freesitemgr
trunk/apps/pyFreenet/freesitemgr.py
Log:
Added undocumented '-s' mode to freesitemgr, which
inserts freesite files one at a time, then does
ClientPutComplexDir to insert a manifest which
contains only redirects to the inserted file CHKs.
This is mainly for debugging purposes.
Modified: trunk/apps/pyFreenet/code.leo
===================================================================
--- trunk/apps/pyFreenet/code.leo 2006-05-17 20:58:53 UTC (rev 8751)
+++ trunk/apps/pyFreenet/code.leo 2006-05-17 21:45:39 UTC (rev 8752)
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<leo_file>
<leo_header file_format="2" tnodes="0" max_tnode_index="8" clone_windows="0"/>
-<globals body_outline_ratio="0.276950043821">
+<globals body_outline_ratio="0.301489921122">
<global_window_position top="51" left="120" height="649" width="1141"/>
<global_log_window_position top="0" left="0" height="0" width="0"/>
</globals>
@@ -36,7 +36,7 @@
<v t="aum.20060506231352.1"><vh>genkey</vh></v>
<v t="aum.20060506231352"><vh>get</vh></v>
<v t="aum.20060507003931"><vh>put</vh></v>
-<v t="aum.20060511001853"><vh>putdir</vh></v>
+<v t="aum.20060511001853" a="V"><vh>putdir</vh></v>
</v>
<v t="aum.20060506224238" a="E"><vh>Other High Level Methods</vh>
<v t="aum.20060514224855"><vh>listenGlobal</vh></v>
@@ -87,7 +87,7 @@
<v t="aum.20060511113333.1"><vh>config</vh></v>
<v t="aum.20060516143534"><vh>globals</vh></v>
<v t="aum.20060511114439" a="E"><vh>class SiteMgr</vh>
-<v t="aum.20060511114439.1" a="V"><vh>__init__</vh></v>
+<v t="aum.20060511114439.1"><vh>__init__</vh></v>
<v t="aum.20060512150118"><vh>__del__</vh></v>
<v t="aum.20060511114439.2"><vh>createConfig</vh></v>
<v t="aum.20060511114604"><vh>loadConfig</vh></v>
@@ -106,7 +106,7 @@
<v t="aum.20060516142202"><vh>run</vh></v>
<v t="aum.20060511120024"><vh>mainline</vh></v>
</v>
-<v t="aum.20060512172707" a="E"
tnodeList="aum.20060512172707,aum.20060512172843,aum.20060512173027,aum.20060512175041,aum.20060512175041.1,aum.20060512175218,aum.20060507155016,aum.20060507162314,aum.20060507162314.2,aum.20060507162314.3,aum.20060507162543.1,aum.20060507195029,aum.20060507163143,aum.20060507154638,aum.20060507195029.1,aum.20060506224545"><vh>@nosent
xmlrpc.py</vh>
+<v t="aum.20060512172707"
tnodeList="aum.20060512172707,aum.20060512172843,aum.20060512173027,aum.20060512175041,aum.20060512175041.1,aum.20060512175218,aum.20060507155016,aum.20060507162314,aum.20060507162314.2,aum.20060507162314.3,aum.20060507162543.1,aum.20060507195029,aum.20060507163143,aum.20060507154638,aum.20060507195029.1,aum.20060506224545"><vh>@nosent
xmlrpc.py</vh>
<v t="aum.20060512172843"><vh>imports</vh></v>
<v t="aum.20060512173027"><vh>globals</vh></v>
<v t="aum.20060512175041" a="E"><vh>class FCPXMLRPCServer</vh>
@@ -635,7 +635,7 @@
try:
k, v = line.split("=")
except:
- #print "unexpected: %s"% line
+ log(ERROR, "_rxMsg: barfed splitting %s" % repr(line))
raise
# attempt int conversion
@@ -6287,6 +6287,10 @@
- name - the name of the freesite, defaults to 'freesite'
- usk - set to True to insert as USK (Default false)
- version - the USK version number, default 0
+
+ - filebyfile - default False - if True, manually inserts
+ each constituent file, then performs the ClientPutComplexDir
+ as a manifest full of redirects
- maxretries - maximum number of retries, default 3
- priority - default 1
@@ -6306,8 +6310,10 @@
Returns:
- the URI under which the freesite can be retrieved
"""
- self._log(INFO, "putdir: uri=%s dir=%s" % (uri, kw['dir']))
+ log = self._log
+ log(INFO, "putdir: uri=%s dir=%s" % (uri, kw['dir']))
+
# -------------------------------------
# format the command
#
@@ -6322,6 +6328,8 @@
version = kw.get('version', 0)
maxretries = kw.get('maxretries', 3)
priority = kw.get('priority', 1)
+ filebyfile = kw.get('filebyfile', False)
+ verbosity = kw.get('verbosity', 0)
id = kw.pop("id", None)
if not id:
@@ -6332,15 +6340,62 @@
if kw.get('usk', False):
uriFull += "%d/" % int(version)
uriFull = uriFull.replace("SSK@", "USK@")
+ while uriFull.endswith("/"):
+ uriFull = uriFull[:-1]
+
+ # hack here - insert as ssk as toad suggests
+ #parts = uriFull.replace("USK@", "SSK@").split("/")
+ #uriFull = "/".join(parts[:-1]) + "-" + parts[-1]
+ #log("putdir: toad hack: URI now is %s" % uriFull)
+ # scan directory and add its files
+ manifest = readdir(kw['dir'])
+
+ manifestDict = {}
+ jobs = []
+ allAtOnce = False
+ if filebyfile:
+ # insert each file, one at a time
+ for file in manifest:
+ relpath = file['relpath']
+ fullpath = file['fullpath']
+ mimetype = file['mimetype']
+
+ manifestDict[relpath] = file
+
+ log(INFO, "Launching insert of %s" % relpath)
+
+ job = self.put("CHK@",
+ file=fullpath,
+ mimetype=mimetype,
+ async=1,
+ verbosity=verbosity,
+ )
+ jobs.append(job)
+ file['job'] = job
+
+ if not allAtOnce:
+ job.wait()
+ log(INFO, "Insert finished for %s" % relpath)
+
+ # wait for jobs to complete
+ if allAtOnce:
+ log(INFO, "Waiting for raw file inserts to finish")
+ while len([j for j in jobs if not j.isComplete()]) > 0:
+ time.sleep(1)
+
+ # all done
+ log(INFO, "All raw files now inserted (or failed)")
+
# build a big command buffer
msgLines = ["ClientPutComplexDir",
"Identifier=%s" % id,
- "Verbosity=%s" % kw.get('verbosity', 0),
+ "Verbosity=%s" % verbosity,
"MaxRetries=%s" % maxretries,
"PriorityClass=%s" % priority,
"URI=%s" % uriFull,
"Persistence=%s" % kw.get("persistence", "connection"),
+ "DefaultName=index.html",
]
if kw.get('Global', False):
@@ -6348,30 +6403,47 @@
else:
msgLines.append("Global=false")
- # scan directory and add its files
+ # add the files
n = 0
- manifest = readdir(kw['dir'])
default = None
for file in manifest:
relpath = file['relpath']
fullpath = file['fullpath']
mimetype = file['mimetype']
+ if filebyfile:
+ if isinstance(file['job'].result, Exception):
+ log(ERROR, "File %s failed to insert" % relpath)
+ continue
+
if relpath == 'index.html':
default = file
self._log(DETAIL, "n=%s relpath=%s" % (repr(n), repr(relpath)))
msgLines.extend(["Files.%d.Name=%s" % (n, relpath),
- "Files.%d.UploadFrom=disk" % n,
- "Files.%d.Filename=%s" % (n, fullpath),
])
+ if filebyfile:
+ msgLines.extend(["Files.%d.UploadFrom=redirect" % n,
+ "Files.%d.TargetURI=%s" % (n, file['job'].result),
+ ])
+ else:
+ msgLines.extend(["Files.%d.UploadFrom=disk" % n,
+ "Files.%d.Filename=%s" % (n, fullpath),
+ ])
n += 1
# now, add the default file
- msgLines.extend(["Files.%d.Name=" % n,
- "Files.%d.UploadFrom=disk" % n,
- "Files.%d.Filename=%s" % (n, default['fullpath']),
- ])
+ if 0:
+ if filebyfile:
+ msgLines.extend(["Files.%d.Name=" % n,
+ "Files.%d.UploadFrom=disk" % n,
+ "Files.%d.Filename=%s" % (n, default['fullpath']),
+ ])
+ else:
+ msgLines.extend(["Files.%d.Name=" % n,
+ "Files.%d.UploadFrom=redirect" % n,
+ "Files.%d.TargetURI=%s" % file['job'].result,
+ ])
msgLines.append("EndMessage")
@@ -6386,10 +6458,7 @@
async=kw.get('async', False),
callback=kw.get('callback', False),
Persistence=kw.get('Persistence', 'connection'),
- )
-
-
-</t>
+ )</t>
<t tx="aum.20060511003500">from fcp import *
n = FCPNode(host="thoth", verbosity=DETAIL)
@@ -6553,12 +6622,15 @@
name=sitename,
version=version,
usk=True,
- verbosity=self.Verbosity)
+ verbosity=self.Verbosity,
+ filebyfile=self.filebyfile)
log(INFO, "site %s updated successfully" % sitename)
except:
traceback.print_exc()
log(ERROR, "site %s failed to update" % sitename)
conf.set(sitename, "hash", hashNew)
+ else:
+ log(INFO, "Site %s not changed, no need to update" % sitename)
self.saveConfig()
@@ -6587,6 +6659,8 @@
- verbosity - logging verbosity level, refer to fcp.node
- fcphost - hostname of fcp, default fcp.node.defaultFCPHost
- fcpport - port number of fcp, default fcp.node.defaultFCPPort
+ - filebyfile - default False - if True, inserts files manually
+ as chks, then builds a manifest full of redirects
"""
# set up the logger
logfile = kw.pop('logfile', sys.stderr)
@@ -6602,6 +6676,8 @@
self.fcpHost = fcpHost
self.fcpPort = fcpPort
+ self.filebyfile = kw.get("filebyfile", False)
+
self.kw = kw
self.node = None
@@ -6625,7 +6701,6 @@
self.config = SafeConfigParser()
self.config.set("DEFAULT", "fcphost", self.fcpHost)
self.config.set("DEFAULT", "fcpport", str(self.fcpPort))
-
</t>
<t tx="aum.20060511114439.2">def createConfig(self, **kw):
"""
@@ -6675,7 +6750,7 @@
try:
self.fcpPort = conf.getint("DEFAULT", "fcpport")
except:
- conf.set("DEFAULT", "fcpport", self.fcpPort)
+ conf.set("DEFAULT", "fcpport", str(self.fcpPort))
for sitename in conf.sections():
@@ -6692,7 +6767,7 @@
self.createConfig()
self.config.set("DEFAULT", "fcphost", self.fcpHost)
- self.config.set("DEFAULT", "fcpport", self.fcpPort)
+ self.config.set("DEFAULT", "fcpport", str(self.fcpPort))
f = file(self.configFile, "a")
@@ -6747,9 +6822,9 @@
opts['name'] = 'freesitemgr'
- print "createNode:"
- print " kw=%s"% kw
- print " opts=%s" % opts
+ #print "createNode:"
+ #print " kw=%s"% kw
+ #print " opts=%s" % opts
#sys.exit(0)
self.node = FCPNode(**opts)
@@ -7902,6 +7977,10 @@
print " - run quietly"
print " -l, --logfile=filename"
print " - location of logfile (default %s)" % logFile
+ print " -s, --single-files"
+ print " - insert one file at a time as CHKs, then insert"
+ print " a manifest which redirects to these, useful"
+ print " for debugging"
print
print "Available Commands:"
print " setup - create/edit freesite config file interactively"
@@ -7920,14 +7999,15 @@
"configfile" : confFile,
"verbosity" : fcp.node.INFO,
"logfile" : logFile,
+ "filebyfile" : False,
}
# process command line switches
try:
cmdopts, args = getopt.getopt(
sys.argv[1:],
- "?hvf:l:",
- ["help", "verbose", "file=", "logfile=",
+ "?hvf:l:s",
+ ["help", "verbose", "file=", "logfile=", "single-files",
]
)
except getopt.GetoptError:
@@ -7955,6 +8035,9 @@
if o in ("-l", "--logfile"):
opts['logfile'] = a
+
+ if o in ("-s", "--single-files"):
+ opts['filebyfile'] = True
# process command
if len(args) < 1:
Modified: trunk/apps/pyFreenet/fcp/node.py
===================================================================
--- trunk/apps/pyFreenet/fcp/node.py 2006-05-17 20:58:53 UTC (rev 8751)
+++ trunk/apps/pyFreenet/fcp/node.py 2006-05-17 21:45:39 UTC (rev 8752)
@@ -426,6 +426,10 @@
- name - the name of the freesite, defaults to 'freesite'
- usk - set to True to insert as USK (Default false)
- version - the USK version number, default 0
+
+ - filebyfile - default False - if True, manually inserts
+ each constituent file, then performs the ClientPutComplexDir
+ as a manifest full of redirects
- maxretries - maximum number of retries, default 3
- priority - default 1
@@ -445,8 +449,10 @@
Returns:
- the URI under which the freesite can be retrieved
"""
- self._log(INFO, "putdir: uri=%s dir=%s" % (uri, kw['dir']))
+ log = self._log
+ log(INFO, "putdir: uri=%s dir=%s" % (uri, kw['dir']))
+
# -------------------------------------
# format the command
#
@@ -461,6 +467,8 @@
version = kw.get('version', 0)
maxretries = kw.get('maxretries', 3)
priority = kw.get('priority', 1)
+ filebyfile = kw.get('filebyfile', False)
+ verbosity = kw.get('verbosity', 0)
id = kw.pop("id", None)
if not id:
@@ -471,15 +479,62 @@
if kw.get('usk', False):
uriFull += "%d/" % int(version)
uriFull = uriFull.replace("SSK@", "USK@")
+ while uriFull.endswith("/"):
+ uriFull = uriFull[:-1]
+
+ # hack here - insert as ssk as toad suggests
+ #parts = uriFull.replace("USK@", "SSK@").split("/")
+ #uriFull = "/".join(parts[:-1]) + "-" + parts[-1]
+ #log("putdir: toad hack: URI now is %s" % uriFull)
+ # scan directory and add its files
+ manifest = readdir(kw['dir'])
+
+ manifestDict = {}
+ jobs = []
+ allAtOnce = False
+ if filebyfile:
+ # insert each file, one at a time
+ for file in manifest:
+ relpath = file['relpath']
+ fullpath = file['fullpath']
+ mimetype = file['mimetype']
+
+ manifestDict[relpath] = file
+
+ log(INFO, "Launching insert of %s" % relpath)
+
+ job = self.put("CHK@",
+ file=fullpath,
+ mimetype=mimetype,
+ async=1,
+ verbosity=verbosity,
+ )
+ jobs.append(job)
+ file['job'] = job
+
+ if not allAtOnce:
+ job.wait()
+ log(INFO, "Insert finished for %s" % relpath)
+
+ # wait for jobs to complete
+ if allAtOnce:
+ log(INFO, "Waiting for raw file inserts to finish")
+ while len([j for j in jobs if not j.isComplete()]) > 0:
+ time.sleep(1)
+
+ # all done
+ log(INFO, "All raw files now inserted (or failed)")
+
# build a big command buffer
msgLines = ["ClientPutComplexDir",
"Identifier=%s" % id,
- "Verbosity=%s" % kw.get('verbosity', 0),
+ "Verbosity=%s" % verbosity,
"MaxRetries=%s" % maxretries,
"PriorityClass=%s" % priority,
"URI=%s" % uriFull,
"Persistence=%s" % kw.get("persistence", "connection"),
+ "DefaultName=index.html",
]
if kw.get('Global', False):
@@ -487,30 +542,47 @@
else:
msgLines.append("Global=false")
- # scan directory and add its files
+ # add the files
n = 0
- manifest = readdir(kw['dir'])
default = None
for file in manifest:
relpath = file['relpath']
fullpath = file['fullpath']
mimetype = file['mimetype']
+ if filebyfile:
+ if isinstance(file['job'].result, Exception):
+ log(ERROR, "File %s failed to insert" % relpath)
+ continue
+
if relpath == 'index.html':
default = file
self._log(DETAIL, "n=%s relpath=%s" % (repr(n), repr(relpath)))
msgLines.extend(["Files.%d.Name=%s" % (n, relpath),
- "Files.%d.UploadFrom=disk" % n,
- "Files.%d.Filename=%s" % (n, fullpath),
])
+ if filebyfile:
+ msgLines.extend(["Files.%d.UploadFrom=redirect" % n,
+ "Files.%d.TargetURI=%s" % (n,
file['job'].result),
+ ])
+ else:
+ msgLines.extend(["Files.%d.UploadFrom=disk" % n,
+ "Files.%d.Filename=%s" % (n, fullpath),
+ ])
n += 1
# now, add the default file
- msgLines.extend(["Files.%d.Name=" % n,
- "Files.%d.UploadFrom=disk" % n,
- "Files.%d.Filename=%s" % (n, default['fullpath']),
- ])
+ if 0:
+ if filebyfile:
+ msgLines.extend(["Files.%d.Name=" % n,
+ "Files.%d.UploadFrom=disk" % n,
+ "Files.%d.Filename=%s" % (n,
default['fullpath']),
+ ])
+ else:
+ msgLines.extend(["Files.%d.Name=" % n,
+ "Files.%d.UploadFrom=redirect" % n,
+ "Files.%d.TargetURI=%s" % file['job'].result,
+ ])
msgLines.append("EndMessage")
@@ -525,10 +597,7 @@
async=kw.get('async', False),
callback=kw.get('callback', False),
Persistence=kw.get('Persistence', 'connection'),
- )
-
-
-
+ )
# high level client methods
def listenGlobal(self, **kw):
@@ -1059,7 +1128,7 @@
try:
k, v = line.split("=")
except:
- #print "unexpected: %s"% line
+ log(ERROR, "_rxMsg: barfed splitting %s" % repr(line))
raise
# attempt int conversion
Modified: trunk/apps/pyFreenet/fcp/sitemgr.py
===================================================================
--- trunk/apps/pyFreenet/fcp/sitemgr.py 2006-05-17 20:58:53 UTC (rev 8751)
+++ trunk/apps/pyFreenet/fcp/sitemgr.py 2006-05-17 21:45:39 UTC (rev 8752)
@@ -35,6 +35,8 @@
- verbosity - logging verbosity level, refer to fcp.node
- fcphost - hostname of fcp, default fcp.node.defaultFCPHost
- fcpport - port number of fcp, default fcp.node.defaultFCPPort
+ - filebyfile - default False - if True, inserts files manually
+ as chks, then builds a manifest full of redirects
"""
# set up the logger
logfile = kw.pop('logfile', sys.stderr)
@@ -50,6 +52,8 @@
self.fcpHost = fcpHost
self.fcpPort = fcpPort
+ self.filebyfile = kw.get("filebyfile", False)
+
self.kw = kw
self.node = None
@@ -187,9 +191,9 @@
opts['name'] = 'freesitemgr'
- print "createNode:"
- print " kw=%s"% kw
- print " opts=%s" % opts
+ #print "createNode:"
+ #print " kw=%s"% kw
+ #print " opts=%s" % opts
#sys.exit(0)
self.node = FCPNode(**opts)
@@ -320,12 +324,15 @@
name=sitename,
version=version,
usk=True,
- verbosity=self.Verbosity)
+ verbosity=self.Verbosity,
+ filebyfile=self.filebyfile)
log(INFO, "site %s updated successfully" % sitename)
except:
traceback.print_exc()
log(ERROR, "site %s failed to update" % sitename)
conf.set(sitename, "hash", hashNew)
+ else:
+ log(INFO, "Site %s not changed, no need to update" % sitename)
self.saveConfig()
Modified: trunk/apps/pyFreenet/freesitemgr
===================================================================
--- trunk/apps/pyFreenet/freesitemgr 2006-05-17 20:58:53 UTC (rev 8751)
+++ trunk/apps/pyFreenet/freesitemgr 2006-05-17 21:45:39 UTC (rev 8752)
@@ -159,6 +159,10 @@
print " - run quietly"
print " -l, --logfile=filename"
print " - location of logfile (default %s)" % logFile
+ print " -s, --single-files"
+ print " - insert one file at a time as CHKs, then insert"
+ print " a manifest which redirects to these, useful"
+ print " for debugging"
print
print "Available Commands:"
print " setup - create/edit freesite config file interactively"
@@ -184,14 +188,15 @@
"configfile" : confFile,
"verbosity" : fcp.node.INFO,
"logfile" : logFile,
+ "filebyfile" : False,
}
# process command line switches
try:
cmdopts, args = getopt.getopt(
sys.argv[1:],
- "?hvf:l:",
- ["help", "verbose", "file=", "logfile=",
+ "?hvf:l:s",
+ ["help", "verbose", "file=", "logfile=", "single-files",
]
)
except getopt.GetoptError:
@@ -219,6 +224,9 @@
if o in ("-l", "--logfile"):
opts['logfile'] = a
+
+ if o in ("-s", "--single-files"):
+ opts['filebyfile'] = True
# process command
if len(args) < 1:
Modified: trunk/apps/pyFreenet/freesitemgr.py
===================================================================
--- trunk/apps/pyFreenet/freesitemgr.py 2006-05-17 20:58:53 UTC (rev 8751)
+++ trunk/apps/pyFreenet/freesitemgr.py 2006-05-17 21:45:39 UTC (rev 8752)
@@ -159,6 +159,10 @@
print " - run quietly"
print " -l, --logfile=filename"
print " - location of logfile (default %s)" % logFile
+ print " -s, --single-files"
+ print " - insert one file at a time as CHKs, then insert"
+ print " a manifest which redirects to these, useful"
+ print " for debugging"
print
print "Available Commands:"
print " setup - create/edit freesite config file interactively"
@@ -184,14 +188,15 @@
"configfile" : confFile,
"verbosity" : fcp.node.INFO,
"logfile" : logFile,
+ "filebyfile" : False,
}
# process command line switches
try:
cmdopts, args = getopt.getopt(
sys.argv[1:],
- "?hvf:l:",
- ["help", "verbose", "file=", "logfile=",
+ "?hvf:l:s",
+ ["help", "verbose", "file=", "logfile=", "single-files",
]
)
except getopt.GetoptError:
@@ -219,6 +224,9 @@
if o in ("-l", "--logfile"):
opts['logfile'] = a
+
+ if o in ("-s", "--single-files"):
+ opts['filebyfile'] = True
# process command
if len(args) < 1:
