Author: aum
Date: 2006-06-08 09:01:04 +0000 (Thu, 08 Jun 2006)
New Revision: 9082
Modified:
trunk/apps/pyFreenet/code.leo
trunk/apps/pyFreenet/fcp/node.py
trunk/apps/pyFreenet/fcp/sitemgr.py
trunk/apps/pyFreenet/freesitemgr
trunk/apps/pyFreenet/freesitemgr.py
trunk/apps/pyFreenet/manpages/freesitemgr.1
trunk/apps/pyFreenet/manpages/freesitemgr.1.html
Log:
Further changes to freesitemgr:
1) Only new/changed files get inserted, unless -i is set
2) site manifests get stored in ~/.freesites
Modified: trunk/apps/pyFreenet/code.leo
===================================================================
--- trunk/apps/pyFreenet/code.leo 2006-06-08 04:42:51 UTC (rev 9081)
+++ trunk/apps/pyFreenet/code.leo 2006-06-08 09:01:04 UTC (rev 9082)
@@ -1,8 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<leo_file>
<leo_header file_format="2" tnodes="0" max_tnode_index="64" clone_windows="0"/>
-<globals body_outline_ratio="0.256694367498">
- <global_window_position top="70" left="78" height="748" width="1083"/>
+<globals body_outline_ratio="0.2594644506">
+ <global_window_position top="38" left="140" height="748" width="1083"/>
<global_log_window_position top="0" left="0" height="0" width="0"/>
</globals>
<preferences/>
@@ -29,7 +29,7 @@
</v>
<v t="aum.20060513073239" a="E"><vh>Package 'fcp'</vh>
<v t="aum.20060516141235" tnodeList="aum.20060516141235"><vh>@nosent
__init__.py</vh></v>
-<v t="aum.20060506215707" a="E"
tnodeList="aum.20060506215707,aum.20060506215707.1,aum.20060506220237,aum.20060506215707.2,aum.20060506215707.3,aum.20060607085345,aum.20060506220237.1,aum.20060506220237.2,aum.20060514223716,aum.20060506231352.1,aum.20060506231352,aum.20060507003931,aum.20060511001853,aum.20060521180804,aum.20060506224238,aum.20060514224855,aum.20060514224919,aum.20060514225725,aum.20060514223936,aum.20060514223822,aum.20060514223845,aum.20060514224020,aum.20060514124642,aum.20060514191601,aum.20060511205201,aum.20060506232639,aum.20060506232639.1,aum.20060511222538,aum.20060512101715,aum.20060511205201.1,aum.20060511205201.2,aum.20060506223545,aum.20060506224238.1,aum.20060506231352.2,aum.20060506220856,aum.20060506222005,aum.20060507124316,aum.20060511103841,aum.20060511103841.1,aum.20060511103952,aum.20060511103952.1,aum.20060604204143,aum.20060514134235,aum.20060512181209,aum.20060514162944,aum.20060514124934,aum.20060512102840,aum.20060514164052,aum.20060509184020.1,aum.20060509184020.2,aum.20060509224119,aum.20060509224221,aum.20060603170554,aum.20060603231840,aum.20060603231840.1,aum.20060603231840.2"><vh>@nosent
node.py</vh>
+<v t="aum.20060506215707" a="E"
tnodeList="aum.20060506215707,aum.20060506215707.1,aum.20060506220237,aum.20060506215707.2,aum.20060506215707.3,aum.20060607085345,aum.20060506220237.1,aum.20060506220237.2,aum.20060514223716,aum.20060506231352.1,aum.20060506231352,aum.20060507003931,aum.20060511001853,aum.20060521180804,aum.20060506224238,aum.20060514224855,aum.20060514224919,aum.20060514225725,aum.20060514223936,aum.20060514223822,aum.20060514223845,aum.20060514224020,aum.20060514124642,aum.20060514191601,aum.20060511205201,aum.20060506232639,aum.20060506232639.1,aum.20060511222538,aum.20060512101715,aum.20060511205201.1,aum.20060511205201.2,aum.20060506223545,aum.20060506224238.1,aum.20060506231352.2,aum.20060506220856,aum.20060506222005,aum.20060507124316,aum.20060511103841,aum.20060511103841.1,aum.20060511103952,aum.20060511103952.1,aum.20060604204143,aum.20060514134235,aum.20060512181209,aum.20060514162944,aum.20060514124934,aum.20060512102840,aum.20060514164052,aum.20060509184020.1,aum.20060509184020.2,aum.20060509224119,aum.20060509224221,aum.20060603170554,aum.20060603231840,aum.20060603231840.1,aum.20060603231840.2"><vh>@file
node.py</vh>
<v t="aum.20060506215707.1"><vh>imports</vh></v>
<v t="aum.20060506220237"><vh>exceptions</vh></v>
<v t="aum.20060506215707.2"><vh>globals</vh></v>
@@ -57,7 +57,7 @@
<v t="aum.20060511205201"><vh>shutdown</vh></v>
</v>
<v t="aum.20060506232639" a="E"><vh>Manager Thread</vh>
-<v t="aum.20060506232639.1"><vh>_mgrThread</vh></v>
+<v t="aum.20060506232639.1" a="V"><vh>_mgrThread</vh></v>
<v t="aum.20060511222538"><vh>_msgIncoming</vh></v>
<v t="aum.20060512101715"><vh>_submitCmd</vh></v>
<v t="aum.20060511205201.1"><vh>_on_rxMsg</vh></v>
@@ -71,7 +71,7 @@
<v t="aum.20060507124316"><vh>_log</vh></v>
</v>
</v>
-<v t="aum.20060511103841" a="E"><vh>class JobTicket</vh>
+<v t="aum.20060511103841"><vh>class JobTicket</vh>
<v t="aum.20060511103841.1"><vh>__init__</vh></v>
<v t="aum.20060511103952"><vh>isComplete</vh></v>
<v t="aum.20060511103952.1"><vh>wait</vh></v>
@@ -516,7 +516,7 @@
</v>
</v>
<v t="aum.20060513073239.5" a="E"><vh>Test files</vh>
-<v t="aum.20060608131556" a="V" tnodeList="aum.20060608131556"><vh>@file
putdirtest.py</vh></v>
+<v t="aum.20060608131556" tnodeList="aum.20060608131556"><vh>@file
putdirtest.py</vh></v>
<v t="aum.20060526123909" tnodeList="aum.20060526123909"><vh>@file
fstest.c</vh></v>
<v t="aum.20060511003500" tnodeList="aum.20060511003500"><vh>@file
test.py</vh></v>
<v t="aum.20060512152233" tnodeList="aum.20060512152233"><vh>@file
genkey.py</vh></v>
@@ -1217,7 +1217,7 @@
log(DEBUG, "No incoming client req")
pass
- self._log(INFO, "Manager thread terminated normally")
+ self._log(DETAIL, "Manager thread terminated normally")
return
except:
@@ -6732,6 +6732,9 @@
# buffer ourselves, not just drop it through as a bunch of keywords,
# since we want to control the order of keyword lines
+ chkonly = False
+ #chkonly = True
+
# get keyword args
dir = kw['dir']
sitename = kw.get('name', 'freesite')
@@ -6767,11 +6770,29 @@
uriFull = uriFull.replace("SSK@", "USK@")
while uriFull.endswith("/"):
uriFull = uriFull[:-1]
+
+ manifestDict = kw.get('manifest', None)
+
+ if manifestDict:
+ # work from the manifest provided by caller
+ #print "got manifest kwd"
+ #print manifestDict
+ manifest = []
+ for relpath, attrDict in manifestDict.items():
+ if attrDict['changed']:
+ attrDict['relpath'] = relpath
+ attrDict['fullpath'] = os.path.join(dir, relpath)
+ manifest.append(attrDict)
+ else:
+ # build manifest by reading the directory
+ #print "no manifest kwd"
+ manifest = readdir(kw['dir'])
+ manifestDict = {}
+ for rec in manifest:
+ manifestDict[rec['relpath']] = rec
+ #print manifestDict
- # scan directory and add its files
- manifest = readdir(kw['dir'])
-
- manifestDict = {}
+
jobs = []
#allAtOnce = False
@@ -6825,7 +6846,7 @@
fullpath = filerec['fullpath']
mimetype = filerec['mimetype']
- manifestDict[relpath] = filerec
+ #manifestDict[relpath] = filerec
log(INFO, "Launching insert of %s" % relpath)
@@ -6839,6 +6860,7 @@
mimetype=mimetype,
async=1,
verbosity=verbosity,
+ chkonly=chkonly,
)
jobs.append(job)
filerec['job'] = job
@@ -6878,6 +6900,9 @@
fullpath = filerec['fullpath']
mimetype = filerec['mimetype']
+ # update the uri in the manifest dict
+ manifestDict[relpath]['uri'] = job.uri
+
# don't add if the file failed to insert
if filebyfile:
if isinstance(filerec['job'].result, Exception):
@@ -6908,13 +6933,16 @@
# --------------------------------------
# now dispatch the job
- finalResult = self._submitCmd(
- id, "ClientPutComplexDir",
- rawcmd=fullbuf,
- async=kw.get('async', False),
- callback=kw.get('callback', False),
- Persistence=kw.get('Persistence', 'connection'),
- )
+ if chkonly:
+ finalResult = "no_uri"
+ else:
+ finalResult = self._submitCmd(
+ id, "ClientPutComplexDir",
+ rawcmd=fullbuf,
+ async=kw.get('async', False),
+ callback=kw.get('callback', False),
+ Persistence=kw.get('Persistence', 'connection'),
+ )
# finally all done, return result or job ticket
return finalResult
@@ -7020,7 +7048,7 @@
# fcp imports
import node
-from node import FCPNode
+from node import FCPNode, guessMimetype
from node import SILENT, FATAL, CRITICAL, ERROR, INFO, DETAIL, DEBUG
</t>
@@ -7046,6 +7074,8 @@
conf = self.config
for sitename in conf.sections():
+
+ print "Updating site '%s'" % sitename
# fill in any incomplete details with site entries
needToSave = False
@@ -7063,6 +7093,7 @@
uri = pub.replace("SSK@", "USK@") + sitename + "/0"
conf.set(sitename, "uri", uri)
conf.set(sitename, "privatekey", priv)
+
if needToSave:
self.saveConfig()
@@ -7073,11 +7104,51 @@
privatekey = conf.get(sitename, "privatekey")
files = node.readdir(dir, gethashes=True)
+
+ # get old manifest - dict of path->{'uri':uri, 'hash':hash} mappings
+ siterec = self.siteManifests[sitename]
+
+ #print "readdir files:"
+ #print files
+ #print "siterec files:"
+ #print siterec
+
+ # determine 'site hash'
h = sha.new()
for f in files:
+ relpath = f['relpath']
+ if siterec.has_key(relpath):
+ filerec = siterec[relpath]
+ filehash = f['hash']
+ if (filerec['hash'] != f['hash']) or self.insertall:
+ #print "File: %s" % relpath
+ #print " oldhash=%s" % filerec['hash']
+ #print " newhash=%s" % f['hash']
+ filerec['changed'] = True
+ else:
+ # new file, create new record and mark as changed
+ #print "new file %s" % relpath
+ siterec[relpath] = {'uri':"",
+ 'hash':f['hash'],
+ 'changed':True,
+ 'fullpath': os.path.join(dir, relpath),
+ 'mimetype': guessMimetype(relpath),
+ }
h.update(f['hash'])
+
+ # dump deleted files from siterec
+ filenames = [f['relpath'] for f in files]
+ for relpath in siterec.keys():
+ if relpath not in filenames:
+ del siterec[relpath]
+
hashNew = h.hexdigest()
- if hashNew != hash:
+
+ # don't insert site if it hasn't changed
+ if (hashNew == hash) and not self.insertall:
+ print "Site '%s' unchanged, skipping update" % sitename
+ else:
+ # site has changed, so better insert it
log(INFO, "Updating site %s" % sitename)
log(INFO, "privatekey=%s" % privatekey)
noSites = False
@@ -7091,20 +7162,33 @@
verbosity=self.Verbosity,
filebyfile=self.filebyfile,
allatonce=self.allatonce,
- maxconcurrent=self.maxconcurrent)
- log(INFO, "site %s updated successfully" % sitename)
+ maxconcurrent=self.maxconcurrent,
+ manifest=siterec,
+ insertall=self.insertall)
+
+ print "Site '%s' updated successfully" % sitename
except:
traceback.print_exc()
- log(ERROR, "site %s failed to update" % sitename)
- conf.set(sitename, "hash", hashNew)
- else:
- log(INFO, "Site %s not changed, no need to update" % sitename)
+ print "site '%s' failed to update" % sitename
+ # update the site's global hash
+ conf.set(sitename, "hash", hashNew)
+
+ # and re-populate the manifest
+ i = 0
+ for relpath, attrdict in siterec.items():
+ #print attrdict
+ conf.set(sitename, "files.%d.relpath" % i, relpath)
+ conf.set(sitename, "files.%d.uri" % i, attrdict['uri'])
+ conf.set(sitename, "files.%d.hash" % i, attrdict['hash'])
+ conf.set(sitename, "files.%d.mimetype" % i, attrdict['mimetype'])
+ i += 1
+
+ # save the updated config
self.saveConfig()
if noSites:
log(INFO, "No sites needed updating")
-
</t>
<t tx="aum.20060511114439">class SiteMgr:
"""
@@ -7135,6 +7219,9 @@
- maxconcurrent - default 10 - if set, this also sets filebyfile
and allatonce both to True. Value of maxconcurrent is the
maximum number of concurrent inserts
+ - insertall - default False - if set, reinserts all files whether
+ they have changed or not. Otherwise, only inserts new or changed
+ files
"""
# set up the logger
logfile = kw.pop('logfile', sys.stderr)
@@ -7147,6 +7234,8 @@
self.verbosity = kw.get('verbosity', 0)
self.Verbosity = kw.get('Verbosity', 0)
+ #print "SiteMgr: verbosity=%s" % self.verbosity
+
self.fcpHost = fcpHost
self.fcpPort = fcpPort
@@ -7165,6 +7254,8 @@
else:
self.maxconcurrent = 10
+ self.insertall = kw.get('insertall', False)
+
self.kw = kw
self.node = None
@@ -7182,6 +7273,9 @@
self.configFile = configFile
+ # a map of sitenames -> filesets
+ self.siteManifests = {}
+
if os.path.isfile(configFile):
self.loadConfig()
else:
@@ -7240,12 +7334,53 @@
conf.set("DEFAULT", "fcpport", str(self.fcpPort))
+ manifests = self.siteManifests
+
for sitename in conf.sections():
if not conf.has_option(sitename, "dir"):
raise Exception("Config file error: No directory specified for
site '%s'" \
% sitename)
+ # get or create a site record dict
+ siterec = manifests.get(sitename, None)
+ if not siterec:
+ siterec = manifests[sitename] = {}
+
+ # grab the manifests
+ for name in conf.options(sitename):
+ #print "name=%s" % name
+
+ if name.startswith("files."):
+ # get or create a file record
+ _ignore, n, attr = name.split(".")
+ n = int(n)
+ val = conf.get(sitename, name)
+
+ filerec = siterec.get(n, None)
+ if not filerec:
+ filerec = siterec[n] = {}
+ filerec[attr] = val
+
+ # now remove from section
+ conf.remove_option(sitename, name)
+
+ # change the siterec to a dict of filename->uri
+ newrec = {}
+ siteDir = conf.get(sitename, "dir")
+ for rec in siterec.values():
+ newrec[rec['relpath']] = {
+ 'fullpath': os.path.join(siteDir, rec['relpath']),
+ 'uri': rec['uri'],
+ 'hash': rec['hash'],
+ 'mimetype' : guessMimetype(rec['relpath']),
+ 'changed' : False,
+ }
+ manifests[sitename] = newrec
+
+ #print manifests
+
+
</t>
<t tx="aum.20060511114604.1">def saveConfig(self):
"""
@@ -7320,6 +7455,8 @@
<t tx="aum.20060511130507">def help():
print "%s: A console-based, cron-able freesite inserter" % sys.argv[0]
+ print "That manages and inserts your freesites as USKs"
+ print
print "Usage: %s" % sys.argv[0]
print "This utility inserts/updates freesites, and is"
@@ -8582,7 +8719,7 @@
"""
dump help info and exit
"""
- print "%s: a console-based freesite insertion utility" % progname
+ print "%s: a console-based USK freesite insertion utility" % progname
print "Usage: %s [options] <command> <args>" % progname
print "Options:"
@@ -8591,7 +8728,7 @@
print " -f, --file=filename"
print " - use a different config file (default is %s)" % confFile
print " -v, --verbose"
- print " - run verbosely"
+ print " - run verbosely, set this twice for even more noise"
print " -q, --quiet"
print " - run quietly"
print " -l, --logfile=filename"
@@ -8612,6 +8749,10 @@
print " limits the number of simultaneous file inserts,"
print " to avoid unduly thrashing the node"
print " setting this option also sets -s and -a"
+ print " -i, --insert-all"
+ print " - if set, force insertion of all files, even ones that"
+ print " aren't new or haven't changed. Otherwise, only insert"
+ print " new/changed files."
print
print "Available Commands:"
print " setup - create/edit freesite config file interactively"
@@ -8628,20 +8769,23 @@
# default job options
opts = {
"configfile" : confFile,
- "verbosity" : fcp.node.INFO,
+ "verbosity" : fcp.node.ERROR,
+ "Verbosity" :1023,
"logfile" : logFile,
"filebyfile" : False,
"allatonce" : False,
"maxconcurrent" : 10,
+ "insertall" : False,
}
# process command line switches
try:
cmdopts, args = getopt.getopt(
sys.argv[1:],
- "?hvf:l:sam:",
+ "?hvf:l:sam:i",
["help", "verbose", "file=", "logfile=",
"single-files", "all-at-once", "max-concurrent=",
+ "insert-all",
]
)
except getopt.GetoptError:
@@ -8658,8 +8802,7 @@
sys.exit(0)
if o in ("-v", "--verbosity"):
- opts['verbosity'] = fcp.node.DETAIL
- opts['Verbosity'] = 1023
+ opts['verbosity'] += 1
if o in ("-q", "--quiet"):
opts['verbosity'] = fcp.node.SILENT
@@ -8679,6 +8822,9 @@
if o in ("-m", "--max-concurrent"):
opts['maxconcurrent'] = int(a)
+ if o in ("-i", "--insert-all"):
+ opts['insertall'] = True
+
# process command
if len(args) < 1:
usage(msg="No command given")
@@ -8872,7 +9018,7 @@
print "No such freesite '%s'" % sitename
return
- if getyesno("Are you sure you wish to delete freesite '%s'", False):
+ if getyesno("Are you sure you wish to delete freesite '%s'" % sitename,
False):
sitemgr.removeSite(sitename)
print "Removed freesite '%s'" % sitename
else:
@@ -13519,9 +13665,9 @@
Generates n random files into a temporary directory
"""
-nfiles = 2
-maxConcurrent = 10
-tmpDir = "/tmp/freesitetest"
+nfiles = 1
+maxConcurrent = 2
+tmpDir = "/tmp/putdirtest"
import sys, os, time
import fcp
Modified: trunk/apps/pyFreenet/fcp/node.py
===================================================================
--- trunk/apps/pyFreenet/fcp/node.py 2006-06-08 04:42:51 UTC (rev 9081)
+++ trunk/apps/pyFreenet/fcp/node.py 2006-06-08 09:01:04 UTC (rev 9082)
@@ -1,4 +1,7 @@
#!/usr/bin/env python
+#@+leo-ver=4
+#@+node:@file node.py
+#@@first
"""
An implementation of a freenet client library for
FCP v2, offering considerable flexibility.
@@ -13,10 +16,14 @@
"""
+#@+others
+#@+node:imports
import sys, os, socket, time, thread
import threading, mimetypes, sha, Queue
import select, traceback, base64
+#@-node:imports
+#@+node:exceptions
class ConnectionRefused(Exception):
"""
cannot connect to given host/port
@@ -49,6 +56,8 @@
class FCPProtocolError(FCPException):
pass
+#@-node:exceptions
+#@+node:globals
# where we can find the freenet node FCP port
defaultFCPHost = "127.0.0.1"
defaultFCPPort = 9481
@@ -83,6 +92,8 @@
defaultVerbosity = ERROR
+#@-node:globals
+#@+node:class FCPNodeConnection
class FCPNode:
"""
Represents an interface to a freenet node via its FCP port,
@@ -127,8 +138,12 @@
calling modes.
"""
+ #@ @+others
+ #@+node:attribs
noCloseSocket = True
+ #@-node:attribs
+ #@+node:__init__
def __init__(self, **kw):
"""
Create a connection object
@@ -198,6 +213,8 @@
self.running = True
thread.start_new_thread(self._mgrThread, ())
+ #@-node:__init__
+ #@+node:__del__
def __del__(self):
"""
object is getting cleaned up, so disconnect
@@ -209,8 +226,12 @@
traceback.print_exc()
pass
+ #@-node:__del__
+ #@+node:FCP Primitives
# basic FCP primitives
+ #@+others
+ #@+node:genkey
def genkey(self, **kw):
"""
Generates and returns an SSK keypair
@@ -246,6 +267,8 @@
return pub, priv
+ #@-node:genkey
+ #@+node:get
def get(self, uri, **kw):
"""
Does a direct get of a key
@@ -339,6 +362,8 @@
# now enqueue the request
return self._submitCmd(id, "ClientGet", **opts)
+ #@-node:get
+ #@+node:put
def put(self, uri="CHK@", **kw):
"""
Inserts a key
@@ -469,6 +494,8 @@
# now dispatch the job
return self._submitCmd(id, "ClientPut", **opts)
+ #@-node:put
+ #@+node:putdir
def putdir(self, uri, **kw):
"""
Inserts a freesite
@@ -521,6 +548,9 @@
# buffer ourselves, not just drop it through as a bunch of keywords,
# since we want to control the order of keyword lines
+ chkonly = False
+ #chkonly = True
+
# get keyword args
dir = kw['dir']
sitename = kw.get('name', 'freesite')
@@ -556,11 +586,29 @@
uriFull = uriFull.replace("SSK@", "USK@")
while uriFull.endswith("/"):
uriFull = uriFull[:-1]
+
+ manifestDict = kw.get('manifest', None)
+
+ if manifestDict:
+ # work from the manifest provided by caller
+ #print "got manifest kwd"
+ #print manifestDict
+ manifest = []
+ for relpath, attrDict in manifestDict.items():
+ if attrDict['changed']:
+ attrDict['relpath'] = relpath
+ attrDict['fullpath'] = os.path.join(dir, relpath)
+ manifest.append(attrDict)
+ else:
+ # build manifest by reading the directory
+ #print "no manifest kwd"
+ manifest = readdir(kw['dir'])
+ manifestDict = {}
+ for rec in manifest:
+ manifestDict[rec['relpath']] = rec
+ #print manifestDict
- # scan directory and add its files
- manifest = readdir(kw['dir'])
-
- manifestDict = {}
+
jobs = []
#allAtOnce = False
@@ -614,7 +662,7 @@
fullpath = filerec['fullpath']
mimetype = filerec['mimetype']
- manifestDict[relpath] = filerec
+ #manifestDict[relpath] = filerec
log(INFO, "Launching insert of %s" % relpath)
@@ -628,6 +676,7 @@
mimetype=mimetype,
async=1,
verbosity=verbosity,
+ chkonly=chkonly,
)
jobs.append(job)
filerec['job'] = job
@@ -667,6 +716,9 @@
fullpath = filerec['fullpath']
mimetype = filerec['mimetype']
+ # update the uri in the manifest dict
+ manifestDict[relpath]['uri'] = job.uri
+
# don't add if the file failed to insert
if filebyfile:
if isinstance(filerec['job'].result, Exception):
@@ -697,17 +749,22 @@
# --------------------------------------
# now dispatch the job
- finalResult = self._submitCmd(
- id, "ClientPutComplexDir",
- rawcmd=fullbuf,
- async=kw.get('async', False),
- callback=kw.get('callback', False),
- Persistence=kw.get('Persistence', 'connection'),
- )
+ if chkonly:
+ finalResult = "no_uri"
+ else:
+ finalResult = self._submitCmd(
+ id, "ClientPutComplexDir",
+ rawcmd=fullbuf,
+ async=kw.get('async', False),
+ callback=kw.get('callback', False),
+ Persistence=kw.get('Persistence', 'connection'),
+ )
# finally all done, return result or job ticket
return finalResult
+ #@-node:putdir
+ #@+node:invertprivate
def invertprivate(self, privatekey):
"""
Converts an SSK or USK private key to a public equivalent
@@ -722,21 +779,31 @@
return uri
+ #@-node:invertprivate
+ #@-others
+ #@-node:FCP Primitives
+ #@+node:Other High Level Methods
# high level client methods
+ #@+others
+ #@+node:listenGlobal
def listenGlobal(self, **kw):
"""
Enable listening on global queue
"""
self._submitCmd(None, "WatchGlobal", Enabled="true", **kw)
+ #@-node:listenGlobal
+ #@+node:ignoreGlobal
def ignoreGlobal(self, **kw):
"""
Stop listening on global queue
"""
self._submitCmd(None, "WatchGlobal", Enabled="false", **kw)
+ #@-node:ignoreGlobal
+ #@+node:purgePersistentJobs
def purgePersistentJobs(self):
"""
Cancels all persistent jobs in one go
@@ -744,30 +811,40 @@
for job in self.getPersistentJobs():
job.cancel()
+ #@-node:purgePersistentJobs
+ #@+node:getAllJobs
def getAllJobs(self):
"""
Returns a list of persistent jobs, excluding global jobs
"""
return self.jobs.values()
+ #@-node:getAllJobs
+ #@+node:getPersistentJobs
def getPersistentJobs(self):
"""
Returns a list of persistent jobs, excluding global jobs
"""
return [j for j in self.jobs.values() if j.isPersistent and not
j.isGlobal]
+ #@-node:getPersistentJobs
+ #@+node:getGlobalJobs
def getGlobalJobs(self):
"""
Returns a list of global jobs
"""
return [j for j in self.jobs.values() if j.isGlobal]
+ #@-node:getGlobalJobs
+ #@+node:getTransientJobs
def getTransientJobs(self):
"""
Returns a list of non-persistent, non-global jobs
"""
return [j for j in self.jobs.values() if not j.isPersistent]
+ #@-node:getTransientJobs
+ #@+node:refreshPersistentRequests
def refreshPersistentRequests(self, **kw):
"""
Sends a ListPersistentRequests to node, to ensure that
@@ -798,12 +875,16 @@
# now enqueue the request
return self._submitCmd(id, "ListPersistentRequests", **opts)
+ #@-node:refreshPersistentRequests
+ #@+node:setVerbosity
def setVerbosity(self, verbosity):
"""
Sets the verbosity for future logging calls
"""
self.verbosity = verbosity
+ #@-node:setVerbosity
+ #@+node:shutdown
def shutdown(self):
"""
Terminates the manager thread
@@ -826,11 +907,17 @@
if self.logfile not in [sys.stdout, sys.stderr]:
self.logfile.close()
+ #@-node:shutdown
+ #@-others
+ #@-node:Other High Level Methods
+ #@+node:Manager Thread
# methods for manager thread
+ #@+others
+ #@+node:_mgrThread
def _mgrThread(self):
"""
This thread is the nucleus of pyfcp, and coordinates incoming
@@ -866,19 +953,23 @@
log(DEBUG, "No incoming client req")
pass
- self._log(INFO, "Manager thread terminated normally")
+ self._log(DETAIL, "Manager thread terminated normally")
return
except:
traceback.print_exc()
self._log(CRITICAL, "manager thread crashed")
+ #@-node:_mgrThread
+ #@+node:_msgIncoming
def _msgIncoming(self):
"""
Returns True if a message is coming in from the node
"""
return len(select.select([self.socket], [], [], pollTimeout)[0]) > 0
+ #@-node:_msgIncoming
+ #@+node:_submitCmd
def _submitCmd(self, id, cmd, **kw):
"""
Submits a command for execution
@@ -923,6 +1014,8 @@
self._log(DETAIL, "Waiting on job")
return job.wait()
+ #@-node:_submitCmd
+ #@+node:_on_rxMsg
def _on_rxMsg(self, msg):
"""
Handles incoming messages from node
@@ -1112,6 +1205,8 @@
job.callback('failed', msg)
job._putResult(FCPException(msg))
return
+ #@-node:_on_rxMsg
+ #@+node:_on_clientReq
def _on_clientReq(self, job):
"""
takes an incoming request job from client and transmits it to
@@ -1131,9 +1226,15 @@
job.reqSentLock.release()
+ #@-node:_on_clientReq
+ #@-others
+ #@-node:Manager Thread
+ #@+node:Low Level Methods
# low level noce comms methods
+ #@+others
+ #@+node:_hello
def _hello(self):
"""
perform the initial FCP protocol handshake
@@ -1145,12 +1246,16 @@
resp = self._rxMsg()
return resp
+ #@-node:_hello
+ #@+node:_getUniqueId
def _getUniqueId(self):
"""
Allocate a unique ID for a request
"""
return "id" + str(int(time.time() * 1000000))
+ #@-node:_getUniqueId
+ #@+node:_txMsg
def _txMsg(self, msgType, **kw):
"""
low level message send
@@ -1204,6 +1309,8 @@
self.socket.send(raw)
+ #@-node:_txMsg
+ #@+node:_rxMsg
def _rxMsg(self):
"""
Receives and returns a message as a dict
@@ -1287,6 +1394,8 @@
# all done
return items
+ #@-node:_rxMsg
+ #@+node:_log
def _log(self, level, msg):
"""
Logs a message. If level > verbosity, don't output it
@@ -1299,7 +1408,13 @@
self.logfile.write(msg)
self.logfile.flush()
+ #@-node:_log
+ #@-others
+ #@-node:Low Level Methods
+ #@-others
+#@-node:class FCPNodeConnection
+#@+node:class JobTicket
class JobTicket:
"""
A JobTicket is an object returned to clients making
@@ -1322,6 +1437,8 @@
- msgs - any messages received from node in connection
to this job
"""
+ #@ @+others
+ #@+node:__init__
def __init__(self, node, id, cmd, kw):
"""
You should never instantiate a JobTicket object yourself
@@ -1358,12 +1475,16 @@
self.reqSentLock = threading.Lock()
self.reqSentLock.acquire()
+ #@-node:__init__
+ #@+node:isComplete
def isComplete(self):
"""
Returns True if the job has been completed
"""
return self.result != None
+ #@-node:isComplete
+ #@+node:wait
def wait(self, timeout=None):
"""
Waits forever (or for a given timeout) for a job to complete
@@ -1373,12 +1494,16 @@
self.lock.release()
return self.getResult()
+ #@-node:wait
+ #@+node:waitTillReqSent
def waitTillReqSent(self):
"""
Waits till the request has been sent to node
"""
self.reqSentLock.acquire()
+ #@-node:waitTillReqSent
+ #@+node:getResult
def getResult(self):
"""
Returns result of job, or None if job still not complete
@@ -1390,6 +1515,8 @@
else:
return self.result
+ #@-node:getResult
+ #@+node:callback
def callback(self, status, value):
"""
This will be replaced in job ticket instances wherever
@@ -1397,6 +1524,8 @@
"""
# no action needed
+ #@-node:callback
+ #@+node:cancel
def cancel(self):
"""
Cancels the job, if it is persistent
@@ -1422,9 +1551,13 @@
Global=isGlobal,
Identifier=self.id)
+ #@-node:cancel
+ #@+node:_appendMsg
def _appendMsg(self, msg):
self.msgs.append(msg)
+ #@-node:_appendMsg
+ #@+node:_putResult
def _putResult(self, result):
"""
Called by manager thread to indicate job is complete,
@@ -1440,6 +1573,8 @@
self.lock.release()
+ #@-node:_putResult
+ #@+node:__repr__
def __repr__(self):
if self.kw.has_key("URI"):
uri = " URI=%s" % self.kw['URI']
@@ -1447,7 +1582,13 @@
uri = ""
return "<FCP job %s:%s%s" % (self.id, self.cmd, uri)
+ #@-node:__repr__
+ #@-others
+#@-node:class JobTicket
+#@+node:util funcs
+#@+others
+#@+node:toBool
def toBool(arg):
try:
arg = int(arg)
@@ -1467,6 +1608,8 @@
else:
return False
+#@-node:toBool
+#@+node:readdir
def readdir(dirpath, prefix='', gethashes=False):
"""
Reads a directory, returning a sequence of file dicts.
@@ -1515,6 +1658,8 @@
return entries
+#@-node:readdir
+#@+node:guessMimetype
def guessMimetype(filename):
"""
Returns a guess of a mimetype based on a filename's extension
@@ -1523,6 +1668,8 @@
if m == None:
m = "text/plain"
return m
+#@-node:guessMimetype
+#@+node:uriIsPrivate
def uriIsPrivate(uri):
"""
analyses an SSK URI, and determines if it is an SSK or USK private key
@@ -1542,7 +1689,11 @@
return False
+#@-node:uriIsPrivate
+#@+node:base64 stuff
# functions to encode/decode base64, freenet alphabet
+#@+others
+#@+node:base64encode
def base64encode(raw):
"""
Encodes a string to base64, using the Freenet alphabet
@@ -1558,6 +1709,8 @@
return enc
+#@-node:base64encode
+#@+node:base64decode
def base64decode(enc):
"""
Decodes a freenet-encoded base64 string back to a binary string
@@ -1575,7 +1728,15 @@
return raw
+#@-node:base64decode
+#@-others
+#@-node:base64 stuff
+#@-others
+#@-node:util funcs
+#@-others
+#@-node:@file node.py
+#@-leo
Modified: trunk/apps/pyFreenet/fcp/sitemgr.py
===================================================================
--- trunk/apps/pyFreenet/fcp/sitemgr.py 2006-06-08 04:42:51 UTC (rev 9081)
+++ trunk/apps/pyFreenet/fcp/sitemgr.py 2006-06-08 09:01:04 UTC (rev 9082)
@@ -8,7 +8,7 @@
# fcp imports
import node
-from node import FCPNode
+from node import FCPNode, guessMimetype
from node import SILENT, FATAL, CRITICAL, ERROR, INFO, DETAIL, DEBUG
fcpHost = node.defaultFCPHost
@@ -43,6 +43,9 @@
- maxconcurrent - default 10 - if set, this also sets filebyfile
and allatonce both to True. Value of maxconcurrent is the
maximum number of concurrent inserts
+ - insertall - default False - if set, reinserts all files whether
+ they have changed or not. Otherwise, only inserts new or changed
+ files
"""
# set up the logger
logfile = kw.pop('logfile', sys.stderr)
@@ -55,6 +58,8 @@
self.verbosity = kw.get('verbosity', 0)
self.Verbosity = kw.get('Verbosity', 0)
+ #print "SiteMgr: verbosity=%s" % self.verbosity
+
self.fcpHost = fcpHost
self.fcpPort = fcpPort
@@ -73,6 +78,8 @@
else:
self.maxconcurrent = 10
+ self.insertall = kw.get('insertall', False)
+
self.kw = kw
self.node = None
@@ -90,6 +97,9 @@
self.configFile = configFile
+ # a map of sitenames -> filesets
+ self.siteManifests = {}
+
if os.path.isfile(configFile):
self.loadConfig()
else:
@@ -156,12 +166,53 @@
conf.set("DEFAULT", "fcpport", str(self.fcpPort))
+ manifests = self.siteManifests
+
for sitename in conf.sections():
if not conf.has_option(sitename, "dir"):
raise Exception("Config file error: No directory specified for
site '%s'" \
% sitename)
+ # get or create a site record dict
+ siterec = manifests.get(sitename, None)
+ if not siterec:
+ siterec = manifests[sitename] = {}
+
+ # grab the manifests
+ for name in conf.options(sitename):
+ #print "name=%s" % name
+
+ if name.startswith("files."):
+ # get or create a file record
+ _ignore, n, attr = name.split(".")
+ n = int(n)
+ val = conf.get(sitename, name)
+
+ filerec = siterec.get(n, None)
+ if not filerec:
+ filerec = siterec[n] = {}
+ filerec[attr] = val
+
+ # now remove from section
+ conf.remove_option(sitename, name)
+
+ # change the siterec to a dict of filename->uri
+ newrec = {}
+ siteDir = conf.get(sitename, "dir")
+ for rec in siterec.values():
+ newrec[rec['relpath']] = {
+ 'fullpath': os.path.join(siteDir, rec['relpath']),
+ 'uri': rec['uri'],
+ 'hash': rec['hash'],
+ 'mimetype' : guessMimetype(rec['relpath']),
+ 'changed' : False,
+ }
+ manifests[sitename] = newrec
+
+ #print manifests
+
+
def saveConfig(self):
"""
Saves the amended config file to disk
@@ -301,6 +352,8 @@
conf = self.config
for sitename in conf.sections():
+
+ print "Updating site '%s'" % sitename
# fill in any incomplete details with site entries
needToSave = False
@@ -318,6 +371,7 @@
uri = pub.replace("SSK@", "USK@") + sitename + "/0"
conf.set(sitename, "uri", uri)
conf.set(sitename, "privatekey", priv)
+
if needToSave:
self.saveConfig()
@@ -328,11 +382,51 @@
privatekey = conf.get(sitename, "privatekey")
files = node.readdir(dir, gethashes=True)
+
+ # get old manifest - dict of path->{'uri':uri, 'hash':hash}
mappings
+ siterec = self.siteManifests[sitename]
+
+ #print "readdir files:"
+ #print files
+ #print "siterec files:"
+ #print siterec
+
+ # determine 'site hash'
h = sha.new()
for f in files:
+ relpath = f['relpath']
+ if siterec.has_key(relpath):
+ filerec = siterec[relpath]
+ filehash = f['hash']
+ if (filerec['hash'] != f['hash']) or self.insertall:
+ #print "File: %s" % relpath
+ #print " oldhash=%s" % filerec['hash']
+ #print " newhash=%s" % f['hash']
+ filerec['changed'] = True
+ else:
+ # new file, create new record and mark as changed
+ #print "new file %s" % relpath
+ siterec[relpath] = {'uri':"",
+ 'hash':f['hash'],
+ 'changed':True,
+ 'fullpath': os.path.join(dir, relpath),
+ 'mimetype': guessMimetype(relpath),
+ }
h.update(f['hash'])
+
+ # dump deleted files from siterec
+ filenames = [f['relpath'] for f in files]
+ for relpath in siterec.keys():
+ if relpath not in filenames:
+ del siterec[relpath]
+
hashNew = h.hexdigest()
- if hashNew != hash:
+
+ # don't insert site if it hasn't changed
+ if (hashNew == hash) and not self.insertall:
+ print "Site '%s' unchanged, skipping update" % sitename
+ else:
+ # site has changed, so better insert it
log(INFO, "Updating site %s" % sitename)
log(INFO, "privatekey=%s" % privatekey)
noSites = False
@@ -346,20 +440,33 @@
verbosity=self.Verbosity,
filebyfile=self.filebyfile,
allatonce=self.allatonce,
- maxconcurrent=self.maxconcurrent)
- log(INFO, "site %s updated successfully" % sitename)
+ maxconcurrent=self.maxconcurrent,
+ manifest=siterec,
+ insertall=self.insertall)
+
+ print "Site '%s' updated successfully" % sitename
except:
traceback.print_exc()
- log(ERROR, "site %s failed to update" % sitename)
- conf.set(sitename, "hash", hashNew)
- else:
- log(INFO, "Site %s not changed, no need to update" % sitename)
+ print "site '%s' failed to update" % sitename
+ # update the site's global hash
+ conf.set(sitename, "hash", hashNew)
+
+ # and re-populate the manifest
+ i = 0
+ for relpath, attrdict in siterec.items():
+ #print attrdict
+ conf.set(sitename, "files.%d.relpath" % i, relpath)
+ conf.set(sitename, "files.%d.uri" % i, attrdict['uri'])
+ conf.set(sitename, "files.%d.hash" % i, attrdict['hash'])
+ conf.set(sitename, "files.%d.mimetype" % i,
attrdict['mimetype'])
+ i += 1
+
+ # save the updated config
self.saveConfig()
if noSites:
log(INFO, "No sites needed updating")
-
def shutdown(self):
self.node.shutdown()
@@ -379,6 +486,8 @@
def help():
print "%s: A console-based, cron-able freesite inserter" % sys.argv[0]
+ print "That manages and inserts your freesites as USKs"
+ print
print "Usage: %s" % sys.argv[0]
print "This utility inserts/updates freesites, and is"
Modified: trunk/apps/pyFreenet/freesitemgr
===================================================================
--- trunk/apps/pyFreenet/freesitemgr 2006-06-08 04:42:51 UTC (rev 9081)
+++ trunk/apps/pyFreenet/freesitemgr 2006-06-08 09:01:04 UTC (rev 9082)
@@ -118,7 +118,7 @@
print "No such freesite '%s'" % sitename
return
- if getyesno("Are you sure you wish to delete freesite '%s'", False):
+ if getyesno("Are you sure you wish to delete freesite '%s'" % sitename,
False):
sitemgr.removeSite(sitename)
print "Removed freesite '%s'" % sitename
else:
@@ -145,7 +145,7 @@
"""
dump help info and exit
"""
- print "%s: a console-based freesite insertion utility" % progname
+ print "%s: a console-based USK freesite insertion utility" % progname
print "Usage: %s [options] <command> <args>" % progname
print "Options:"
@@ -154,7 +154,7 @@
print " -f, --file=filename"
print " - use a different config file (default is %s)" % confFile
print " -v, --verbose"
- print " - run verbosely"
+ print " - run verbosely, set this twice for even more noise"
print " -q, --quiet"
print " - run quietly"
print " -l, --logfile=filename"
@@ -175,6 +175,10 @@
print " limits the number of simultaneous file inserts,"
print " to avoid unduly thrashing the node"
print " setting this option also sets -s and -a"
+ print " -i, --insert-all"
+ print " - if set, force insertion of all files, even ones that"
+ print " aren't new or haven't changed. Otherwise, only insert"
+ print " new/changed files."
print
print "Available Commands:"
print " setup - create/edit freesite config file interactively"
@@ -198,20 +202,23 @@
# default job options
opts = {
"configfile" : confFile,
- "verbosity" : fcp.node.INFO,
+ "verbosity" : fcp.node.ERROR,
+ "Verbosity" :1023,
"logfile" : logFile,
"filebyfile" : False,
"allatonce" : False,
"maxconcurrent" : 10,
+ "insertall" : False,
}
# process command line switches
try:
cmdopts, args = getopt.getopt(
sys.argv[1:],
- "?hvf:l:sam:",
+ "?hvf:l:sam:i",
["help", "verbose", "file=", "logfile=",
"single-files", "all-at-once", "max-concurrent=",
+ "insert-all",
]
)
except getopt.GetoptError:
@@ -228,8 +235,7 @@
sys.exit(0)
if o in ("-v", "--verbosity"):
- opts['verbosity'] = fcp.node.DETAIL
- opts['Verbosity'] = 1023
+ opts['verbosity'] += 1
if o in ("-q", "--quiet"):
opts['verbosity'] = fcp.node.SILENT
@@ -249,6 +255,9 @@
if o in ("-m", "--max-concurrent"):
opts['maxconcurrent'] = int(a)
+ if o in ("-i", "--insert-all"):
+ opts['insertall'] = True
+
# process command
if len(args) < 1:
usage(msg="No command given")
Modified: trunk/apps/pyFreenet/freesitemgr.py
===================================================================
--- trunk/apps/pyFreenet/freesitemgr.py 2006-06-08 04:42:51 UTC (rev 9081)
+++ trunk/apps/pyFreenet/freesitemgr.py 2006-06-08 09:01:04 UTC (rev 9082)
@@ -118,7 +118,7 @@
print "No such freesite '%s'" % sitename
return
- if getyesno("Are you sure you wish to delete freesite '%s'", False):
+ if getyesno("Are you sure you wish to delete freesite '%s'" % sitename,
False):
sitemgr.removeSite(sitename)
print "Removed freesite '%s'" % sitename
else:
@@ -145,7 +145,7 @@
"""
dump help info and exit
"""
- print "%s: a console-based freesite insertion utility" % progname
+ print "%s: a console-based USK freesite insertion utility" % progname
print "Usage: %s [options] <command> <args>" % progname
print "Options:"
@@ -154,7 +154,7 @@
print " -f, --file=filename"
print " - use a different config file (default is %s)" % confFile
print " -v, --verbose"
- print " - run verbosely"
+ print " - run verbosely, set this twice for even more noise"
print " -q, --quiet"
print " - run quietly"
print " -l, --logfile=filename"
@@ -175,6 +175,10 @@
print " limits the number of simultaneous file inserts,"
print " to avoid unduly thrashing the node"
print " setting this option also sets -s and -a"
+ print " -i, --insert-all"
+ print " - if set, force insertion of all files, even ones that"
+ print " aren't new or haven't changed. Otherwise, only insert"
+ print " new/changed files."
print
print "Available Commands:"
print " setup - create/edit freesite config file interactively"
@@ -198,20 +202,23 @@
# default job options
opts = {
"configfile" : confFile,
- "verbosity" : fcp.node.INFO,
+ "verbosity" : fcp.node.ERROR,
+ "Verbosity" :1023,
"logfile" : logFile,
"filebyfile" : False,
"allatonce" : False,
"maxconcurrent" : 10,
+ "insertall" : False,
}
# process command line switches
try:
cmdopts, args = getopt.getopt(
sys.argv[1:],
- "?hvf:l:sam:",
+ "?hvf:l:sam:i",
["help", "verbose", "file=", "logfile=",
"single-files", "all-at-once", "max-concurrent=",
+ "insert-all",
]
)
except getopt.GetoptError:
@@ -228,8 +235,7 @@
sys.exit(0)
if o in ("-v", "--verbosity"):
- opts['verbosity'] = fcp.node.DETAIL
- opts['Verbosity'] = 1023
+ opts['verbosity'] += 1
if o in ("-q", "--quiet"):
opts['verbosity'] = fcp.node.SILENT
@@ -249,6 +255,9 @@
if o in ("-m", "--max-concurrent"):
opts['maxconcurrent'] = int(a)
+ if o in ("-i", "--insert-all"):
+ opts['insertall'] = True
+
# process command
if len(args) < 1:
usage(msg="No command given")
Modified: trunk/apps/pyFreenet/manpages/freesitemgr.1
===================================================================
--- trunk/apps/pyFreenet/manpages/freesitemgr.1 2006-06-08 04:42:51 UTC (rev
9081)
+++ trunk/apps/pyFreenet/manpages/freesitemgr.1 2006-06-08 09:01:04 UTC (rev
9082)
@@ -37,13 +37,36 @@
\fB\-s\fR, \fB\-\-single\-files\fR
Insert each file of the freesite as single CHKs, and redirect
-to these from within the freesite manifest
+to these from within the freesite manifest. You \fBmust\fR use
+this option if inserting through an FCP port not on your local machine.
+Note that files will be inserted one at a time, with each next file
+not being inserted till the previous one is completely inserted. If you
+wish to insert more than one file concurrently, use the \fB\-a\fR
+or \fB\-m\fR options.
+.TP
+\fB\-a\fR, \fB\-\-all\-at\-once\fR
+Enables concurrent insert of the site's files. By default, a maximum
+of 10 files will be inserting at any one time. You can increase this with
+\fB\-m\fR. Note that this \fB\-a\fR option automatically turns on
+\fB\-s\fR.
+.TP
+
+\fB\-m\fR, \fB\-\-max\-concurrent\-inserts=\fR\fInum\fR
+Turns on \fB\-s\fR and \fB\-a\fR, and sets the maximum number
+of concurrent file inserts to \fInum\fR. You can set a very high
+value if you want, but it will be of little use because the node
+itself regulates how many concurrent inserts it performs.
+.TP
+
+\fB\-i\fR, \fB\-\-insert\-all\fR
+Forces insert of all files in all aites. Otherwise, sites with no new
+or changed files will be skipped, and in sites that have changed,
+only the new/changed files (and a fresh manifest) will be inserted.
+
.LP
.SH "COMMANDS"
-.LP
-
\fBsetup\fP
Create or modify the freesite config file (\fB~/.freesites\fP)
interactively.
Modified: trunk/apps/pyFreenet/manpages/freesitemgr.1.html
===================================================================
--- trunk/apps/pyFreenet/manpages/freesitemgr.1.html 2006-06-08 04:42:51 UTC
(rev 9081)
+++ trunk/apps/pyFreenet/manpages/freesitemgr.1.html 2006-06-08 09:01:04 UTC
(rev 9082)
@@ -53,7 +53,29 @@
<DT><DD>
<B>-s</B>, <B>--single-files</B>
Insert each file of the freesite as single CHKs, and redirect
-to these from within the freesite manifest
+to these from within the freesite manifest. You <B>must</B> use
+this option if inserting through an FCP port not on your local machine.
+Note that files will be inserted one at a time, with each next file
+not being inserted till the previous one is completely inserted. If you
+wish to insert more than one file concurrently, use the <B>-a</B>
+or <B>-m</B> options.
+<DT><DD>
+<B>-a</B>, <B>--all-at-once</B>
+Enables concurrent insert of the site's files. By default, a maximum
+of 10 files will be inserting at any one time. You can increase this with
+<B>-m</B>. Note that this <B>-a</B> option automatically turns on
+<B>-s</B>.
+<DT><DD>
+<B>-m</B>, <B>--max-concurrent-inserts=</B><I>num</I>
+Turns on <B>-s</B> and <B>-a</B>, and sets the maximum number
+of concurrent file inserts to <I>num</I>. You can set a very high
+value if you want, but it will be of little use because the node
+itself regulates how many concurrent inserts it performs.
+<DT><DD>
+<B>-i</B>, <B>--insert-all</B>
+Forces insert of all files in all aites. Otherwise, sites with no new
+or changed files will be skipped, and in sites that have changed,
+only the new/changed files (and a fresh manifest) will be inserted.
<P>
</DL>
<P>
@@ -62,9 +84,6 @@
<A NAME="lbAF"> </A>
<H2>COMMANDS</H2>
-<P>
-
-<P>
<B>setup</B>
Create or modify the freesite config file (<B>~/.freesites</B>)
interactively.
@@ -135,6 +154,6 @@
This document was created by
<A HREF="/cgi-bin/man/man2html">man2html</A>,
using the manual pages.<BR>
-Time: 01:47:06 GMT, May 28, 2006
+Time: 08:59:41 GMT, June 08, 2006
</BODY>
</HTML>
