Author: nextgens
Date: 2006-06-09 22:06:34 +0000 (Fri, 09 Jun 2006)
New Revision: 9112
Modified:
branches/legacy/apps/yafi/Changelog
branches/legacy/apps/yafi/fcp.py
branches/legacy/apps/yafi/fec.py
branches/legacy/apps/yafi/freenet.py
branches/legacy/apps/yafi/metadata.py
branches/legacy/apps/yafi/yafi
Log:
updated yafi
Modified: branches/legacy/apps/yafi/Changelog
===================================================================
--- branches/legacy/apps/yafi/Changelog 2006-06-09 21:47:38 UTC (rev 9111)
+++ branches/legacy/apps/yafi/Changelog 2006-06-09 22:06:34 UTC (rev 9112)
@@ -32,3 +32,9 @@
- The FCPSocket now buffers properly; it no longer sucks
- Colorized console output
+20050126:
+- Make output more colorful and informative
+- Fixed unnamed redirect bug
+- Fixed healing for redirected splitfiles
+- Added splitfileinfo command
+
Modified: branches/legacy/apps/yafi/fcp.py
===================================================================
--- branches/legacy/apps/yafi/fcp.py 2006-06-09 21:47:38 UTC (rev 9111)
+++ branches/legacy/apps/yafi/fcp.py 2006-06-09 22:06:34 UTC (rev 9112)
@@ -81,11 +81,6 @@
for k in hexvals:
dict[k] = int(dict[k], 16)
- if dict['IsTransient'] == 'true':
- dict['IsTransient'] = True
- else:
- dict['IsTransient'] = False
-
return dict
def get(self, key, htl, removelocal=False):
@@ -181,12 +176,20 @@
self._sock.readline() # the 'Data' line
metaleft = info['MetadataLength'] - metarcvd
if metaleft <= chunklength:
- metadata += self._sock.recvbytes(metaleft)
+ newmetadata = self._sock.recvbytes(metaleft)
+ if newmetadata is None:
+ break
+ metadata += newmetadata
metarcvd += metaleft
newdata = self._sock.recvbytes(chunklength -
metaleft)
+ if newdata is None:
+ break
data += newdata
else:
- metadata += self._sock.recvbytes(chunklength)
+ newmetadata = self._sock.recvbytes(chunklength)
+ if newmetadata is None:
+ break
+ metadata += newmetadata
metarcvd += chunklength
bytesrcvd += chunklength
self._setstatus('Transferring', bytesrcvd)
@@ -219,6 +222,8 @@
length = int(((line.strip()).split('='))[1], 16)
self._sock.readline() # the 'Data' line
newdata = self._sock.recvbytes(length)
+ if newdata is None:
+ break
data += newdata
bytesrcvd += length
self._setstatus('Transferring', bytesrcvd)
@@ -492,7 +497,7 @@
line = self._sock.readline().strip()
URI = (line.split('='))[1].lstrip('freenet:')
self._sock.close()
- raise KeyCollisionError, URI
+ raise KeyCollisionError(URI)
if err == 'SizeError':
self._setstatus('Stopped')
self._sock.close()
@@ -564,45 +569,61 @@
class Error(Exception):
"""Base class for errors in the FCP module."""
- pass
+ def __init__(self, message='An unknown error occured in FCP processing.'):
+ self._errormessage = message
+ def getErrorMessage(self):
+ return self._errormessage
+
class ConnectError(Error):
"""Unable to connect to FCP socket.
args is the (host, port) tuple that failed.
"""
- pass
+ def __init__(self, host, port):
+ Error.__init__(self, 'Unable to connect to %s:%s' % (host, port))
class FormatError(Error):
- """Error in message format."""
- pass
+ def __init__(self):
+ Error.__init__(self, 'Error in message format!')
class FailedError(Error):
- """Error in the node itself."""
- pass
+ def __init__(self):
+ Error.__init__(self, 'Internal error in Freenet node!')
class URIError(Error):
- pass
+ def __init__(self):
+ Error.__init__(self, 'Error in Freenet URI!')
class DNFError(Error):
- pass
+ def __init__(self):
+ Error.__init__(self, 'Data not found!')
class TimeoutError(Error):
- pass
+ def __init__(self):
+ Error.__init__(self, 'Request timed out!')
class KeyCollisionError(Error):
- pass
+ def __init__(self, key):
+ Error.__init__(self, 'Key collision: %s' % key)
+ self._URI = key
+ def getURI(self):
+ return self._URI
+
class SizeError(Error):
- pass
+ def __init__(self):
+ Error.__init__(self, 'Key size error!')
class RNFError(Error):
"""Route Not Found.
Atributes are Unreachable, Restarted, and Rejected.
"""
- pass
+ def __init__(self):
+ Error.__init__(self, 'Route not found!')
+
class FCPSocket(socket.socket):
"""FCP Socket: does some magic FCP stuff, as well as buffering and
timeouts. You probably shouldn't use this for anything other than YAFI's
@@ -629,7 +650,7 @@
try:
self.connect((host, port))
except:
- raise ConnectError(self.host, self.port)
+ raise ConnectError(host, port)
self.send(self._magicstring)
def readline(self):
Modified: branches/legacy/apps/yafi/fec.py
===================================================================
--- branches/legacy/apps/yafi/fec.py 2006-06-09 21:47:38 UTC (rev 9111)
+++ branches/legacy/apps/yafi/fec.py 2006-06-09 22:06:34 UTC (rev 9112)
@@ -1,13 +1,23 @@
import onionfec_a_1_2
+DEFAULT_FEC = 'OnionFEC_a_1_2'
algolist = {'OnionFEC_a_1_2': onionfec_a_1_2}
def FEC(algoname):
if algoname in algolist:
return algolist[algoname].FEC()
else:
- raise FECError, 'Unknown FEC algorithm.'
+ raise FECError
+class Error(Exception):
+ """Base class for errors in the FEC module."""
+ def __init__(self, message='An unknown error occured in FCP processing.'):
+ self._errormessage = message
+
+ def getErrorMessage(self):
+ return self._errormessage
+
class FECError(Exception):
- pass
+ def __init__(self):
+ Error.__init__(self, 'Unknown FEC error!')
Modified: branches/legacy/apps/yafi/freenet.py
===================================================================
--- branches/legacy/apps/yafi/freenet.py 2006-06-09 21:47:38 UTC (rev
9111)
+++ branches/legacy/apps/yafi/freenet.py 2006-06-09 22:06:34 UTC (rev
9112)
@@ -16,7 +16,7 @@
import mimedata
import fcp
-from fcp import DNFError, RNFError, FormatError, URIError, TimeoutError,
FailedError, KeyCollisionError
+from fcp import DNFError, RNFError, FormatError, URIError, TimeoutError,
FailedError, KeyCollisionError, ConnectError
import fec
from fec import FECError
import metadata as metadatalib
@@ -24,7 +24,6 @@
MAXHTL = 20
MAXSIZE = 0x100000 # Max size for a CHK block (1 MiB)
-DEFAULT_FEC = 'OnionFEC_a_1_2'
def _shuffle(lst):
l = len(lst)
@@ -51,7 +50,8 @@
self._threadpool = [] #Used to pass around splitfile status
def get(self, key, aggressive=False, htl=MAXHTL, threads=15,
- healingthreads=10, retries=2, removelocal=False):
+ healingthreads=10, retries=2, removelocal=False,
+ splitfile=True, raw=False):
"""Get a piece of data from Freenet.
"""
self._aggressive=aggressive
@@ -60,9 +60,13 @@
self._nthreads = threads
self._hthreads = healingthreads
self._removelocal = removelocal
+ self._splitfile = splitfile
self._successblocks = []
- return self._get(key)
+ if raw:
+ return self._rawget(key)
+ else:
+ return self._get(key)
def put(self, data, metadata='', htl=MAXHTL, removelocal=True,
privkey=None, cryptokey='', name=None, threads=5, aggressive=True,
@@ -95,16 +99,19 @@
while True:
try:
- metadata = (self._f.get(key, htl=self._htl,
removelocal=False))[0]
+ self.get(key, htl=self._htl, removelocal=False,
+ splitfile=False)
except (DNFError, RNFError, TimeoutError):
continue
- break
+ except SplitFileError, err:
+ metadata = err.getMetadata()
+ break
docs = metadatalib.parse(metadata)
nblocks = int(docs[0]['SplitFile.BlockCount'], 16)
ncheckblocks = int(docs[0]['SplitFile.CheckBlockCount'], 16)
- encoder = fec.FEC(DEFAULT_FEC)
+ encoder = fec.FEC(fec.DEFAULT_FEC)
blocks, checkblocks = encoder.encode(data)
del data
@@ -134,7 +141,7 @@
self._putblocks(allblocks, threads=self._hthreads)
self._setstatus('Stopped')
- def rawget(self, key, htl=MAXHTL):
+ def _rawget(self, key, htl=MAXHTL):
self._setstatus('Active')
try:
metadata, data = self._f.get(key, htl=htl)
@@ -150,9 +157,10 @@
filename = key[i + 2:]
else:
filename = None
-
+
try:
- metadata, data = self._f.get(key, htl=self._htl,
removelocal=self._removelocal)
+ metadata, data = self._f.get(key, htl=self._htl,
+ removelocal=self._removelocal)
except fcp.Error:
self._setstatus('Stopped')
raise
@@ -161,18 +169,22 @@
if filename is None:
filename = ''
- if data.startswith(zipfile.stringFileHeader): # We have a
container site.
+ if data.startswith(zipfile.stringFileHeader):
+ # We have a container site.
return self._getfromzip(data, filename)
- else: # A regular redirect.
+ else:
+ # A regular redirect.
return self._getdoc(filename, metadata, data)
- else: # Finally, no redirects!
+ else:
+ # Finally, no redirects!
self._setstatus('Stopped')
return data
def _get_dbr(self, key, filename, increment, offset):
key = metadatalib.dbrkey(key, increment, offset)
try:
- metadata, data = self._f.get(key, htl=self._htl,
removelocal=self._removelocal)
+ metadata, data = self._f.get(key, htl=self._htl,
+ removelocal=self._removelocal)
except fcp.Error:
self._setstatus('Stopped')
raise
@@ -187,7 +199,7 @@
try:
data = z.read(filename)
except KeyError:
- raise KeyNotFoundError, filename
+ raise KeyNotFoundError(filename)
z.close()
t.close()
os.remove(n)
@@ -199,8 +211,8 @@
"""
documents = metadatalib.parse(metadata)
if documents == [] and filename == '':
- # Some real dipshit has inserted a file with no redirects when we
were expecting one
- # data is probably empty
+ # Some real dipshit has inserted a file with no redirects when we
+ # were expecting one; data is probably empty
return data
for doc in documents:
@@ -214,19 +226,25 @@
doc.setdefault('DateRedirect.Increment', 86400),
doc.setdefault('DateRedirect.Offset', 0))
else:
- return self._getsplitfile(doc)
+ if self._splitfile:
+ return self._getsplitfile(doc)
+ else:
+ raise SplitFileError(metadata)
# The specified filename wasn't among the redirects; look for an
unnamed redirect.
for doc in documents:
+ if doc.has_key('Redirect.Target') and doc['Name'] == '':
+ self._setstatus('Redirect')
+ return self._get(doc['Redirect.Target'] + '//' + filename)
if doc.has_key('DateRedirect.Target') and doc['Name'] == '':
self._setstatus('DBR')
return self._get_dbr(doc['DateRedirect.Target'], filename,
- doc.setdefault('DateRedirect.Increment', 86400),
- doc.setdefault('DateRedirect.Offset', 0))
+ doc.setdefault('DateRedirect.Increment', 86400),
+ doc.setdefault('DateRedirect.Offset', 0))
# We can't find the specified key in the manifest; we're hosed.
self._setstatus('Stopped')
- raise KeyNotFoundError, filename
+ raise KeyNotFoundError(filename)
def _getsplitfile(self, doc):
def hexnums(n):
@@ -381,7 +399,7 @@
self.close()
if len(doneblocks) < nblocks:
- raise SegmentFailedError, (segment, len(doneblocks), nblocks)
+ raise SegmentFailedError(segment, len(doneblocks), nblocks)
return doneblocks
@@ -413,7 +431,7 @@
self._setstatus('Encoding')
- encoder = fec.FEC(DEFAULT_FEC)
+ encoder = fec.FEC(fec.DEFAULT_FEC)
checksum = sha(data).hexdigest()
fecfile = fectempfile(checksum, self._tmpdir)
@@ -641,7 +659,7 @@
self._reqq.put((None, None, None))
while len(self._threadpool) > 0:
for t in self._threadpool:
- t.join(1)
+ t.isAlive() and t.join(1)
self._threadpool.remove(t)
@@ -720,20 +738,37 @@
return (self._index, self._f.getstatus())
class Error(Exception):
- pass
+ def __init__(self, message='An unknown error occured in Freenet
processing.'):
+ self._errormessage = message
+ def getErrorMessage(self):
+ return self._errormessage
+
class KeyNotFoundError(Error):
- pass
+ def __init__(self, filename):
+ Error.__init__(self, 'The key %s was not found in the manifest!' %
filename)
class SegmentFailedError(Error):
- pass
+ def __init__(self, segment, doneblocks, totalblocks):
+ Error.__init__(self, 'Splitfile segment %d failed with %d of %d
blocks' % \
+ (segment + 1, doneblocks, totalblocks))
class HealingError(Error):
- pass
+ def __init__(self):
+ Error.__init__(self, 'Healing failed!')
class ChecksumFailedError(Error):
- pass
+ def __init__(self):
+ Error.__init__(self, 'Checksum failed!')
+class SplitFileError(Error):
+ def __init__(self, metadata=''):
+ Error.__init__(self, 'A splitfile was encountered with splitfile
downloading disabled!')
+ self._metadata = metadata
+
+ def getMetadata(self):
+ return self._metadata
+
def debugwrite(s):
sys.stderr.write("%s:\t%s\n" % (threading.currentThread().getName(), s))
sys.stderr.flush()
Modified: branches/legacy/apps/yafi/metadata.py
===================================================================
--- branches/legacy/apps/yafi/metadata.py 2006-06-09 21:47:38 UTC (rev
9111)
+++ branches/legacy/apps/yafi/metadata.py 2006-06-09 22:06:34 UTC (rev
9112)
@@ -66,6 +66,9 @@
def dbrkey(target, increment=86400, offset=0, future=0):
"""Generate a DBR key.
+
+ i.e. SSK at
ETERJ~iiRJvzKfgl-oAxykDx21EPAgM,wUizNX7UGxMzyUwjJdtDCg/SonaxFlog ->
+ SSK at
ETERJ~iiRJvzKfgl-oAxykDx21EPAgM,wUizNX7UGxMzyUwjJdtDCg/41a91500-SonaxFlog
"""
def dbrdate(increment, offset, future):
curtime = int(time.time())
Modified: branches/legacy/apps/yafi/yafi
===================================================================
--- branches/legacy/apps/yafi/yafi 2006-06-09 21:47:38 UTC (rev 9111)
+++ branches/legacy/apps/yafi/yafi 2006-06-09 22:06:34 UTC (rev 9112)
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python -u
"""yafi
@@ -16,14 +16,20 @@
import freenet
import fcp
import fec
+import metadata as metadatalib
versionstring = "Yet Another Freenet Interface"
usagestring = "%prog <command> [options] [URI] ..."
-commands = ['put', 'get', 'info', 'chk', 'svk']
-
globaldata = None
+successcolor = 'green'
+failedcolor = 'red'
+skippedcolor = 'lblue'
+splitfilecolor = 'yellow'
+
+DEFAULTTHREADS = 10
+
def main(argv=[__name__]):
if len(argv) < 2:
sys.exit(1)
@@ -89,6 +95,9 @@
parser.add_option('--healing', dest='healpercent', type='int', default=100,
metavar='PERCENT', help='Percent of healing blocks to ' \
'upload')
+ parser.add_option('--nosplitfile', dest='splitfile', default=True,
+ action='store_false', help="do'nt follow splitfile
redirects")
+
return parser
def get_command(options, keys):
@@ -96,13 +105,8 @@
#default number of threads
if options.threads is None:
- options.threads = 20
+ options.threads = DEFAULTTHREADS
- #sanitize the keys
- for i in range(len(keys)):
- keys[i] = keys[i].strip('"')
- keys[i] = urllib.unquote(keys[i])
-
if len(keys) > 1:
if options.filename is not None:
options.filename = None
@@ -110,16 +114,27 @@
if options.infile is not None:
if options.infile == '-':
- print "Reading keys from standard input:"
+ print "Reading keys from standard input..."
infile = sys.stdin
else:
- print "Reading keys from %s:" % options.infile
- infile = open(options.infile)
+ print "Reading keys from %s..." % options.infile
+ try:
+ infile = open(options.infile)
+ except IOError:
+ print "Input file not found!"
+ return 2
for line in infile.readlines():
if line[:3] == 'CHK' or line[:3] == 'KSK' or line[:3] == 'SSK':
keys.append(line.strip())
+ #sanitize the keys
+ for i in range(len(keys)):
+ keys[i] = keys[i].strip('"')
+ keys[i] = urllib.unquote(keys[i])
+
+ print 'Fetching %d keys:' % len(keys)
+
for key in keys:
if (options.filename is None) and \
(key[:3] != 'KSK') and \
@@ -129,7 +144,8 @@
return 2
if options.dir is not None:
- options.dir = os.path.dirname(options.dir)
+ if options.dir.endswith('/'):
+ options.dir = os.path.dirname(options.dir)
try:
os.stat(options.dir)
except OSError:
@@ -139,7 +155,12 @@
e = threading.Event()
else:
e = None
- n = freenet.Node(options.host, options.port, statevent=e)
+ try:
+ n = freenet.Node(options.host, options.port, statevent=e)
+ except freenet.ConnectError, err:
+ print err.getErrorMessage()
+ return 1
+
statthread = StatThread(e, n, direction='In')
statthread.start()
@@ -148,9 +169,11 @@
successkeys = []
failedkeys = []
skippedkeys = []
+ splitfilekeys = []
try:
for key in keys:
+ currentkey = len(successkeys) + len(failedkeys) + len(skippedkeys)
+ len(splitfilekeys) + 1
if options.filename != None:
if options.filename == '-':
filename = sys.stdout
@@ -164,65 +187,86 @@
filename = options.dir + '/' + filename
if filename is not sys.stdout:
- print 'Saving', key, 'to', filename, '...'
+ print 'Saving',
+ if options.raw:
+ print 'metadata for',
+ print key, 'to', filename, '...'
if not options.overwrite:
try:
os.stat(filename)
except OSError:
pass
else:
- print "%s already exists! Skipping download..." % \
- filename
+ print color('%s already exists! Skipping download...'
% \
+ filename, skippedcolor), '(Key %d of %d)' %
(currentkey, len(keys))
skippedkeys.append(key)
continue
elif options.verbose is True:
- print "Writing", key, "to stdout..."
+ print 'Writing',
+ if options.raw:
+ print 'metadata for',
+ else:
+ print ' ',
+ print key, 'to stdout...'
try:
- if not options.raw:
- data = n.get(key, htl=options.htl, retries=options.retries,
- threads=options.threads,
- aggressive=options.aggressive)
- else:
- data = n.rawget(key, htl=options.htl)
- except freenet.DNFError:
- print 'Data not found!'
+ data = n.get(key, htl=options.htl, retries=options.retries,
+ threads=options.threads,
aggressive=options.aggressive,
+ splitfile=options.splitfile, raw=options.raw)
+ except freenet.DNFError, err:
+ print color(err.getErrorMessage(), failedcolor), '(Key %d of
%d)' % (currentkey, len(keys))
failedkeys.append(key)
- except freenet.RNFError:
- print 'Route not found!'
+ except freenet.RNFError, err:
+ print color(err.getErrorMessage(), failedcolor), '(Key %d of
%d)' % (currentkey, len(keys))
failedkeys.append(key)
except freenet.FECError, err:
- print "FEC Error:", err[0]
+ print color(err.getErrorMessage(), failedcolor), '(Key %d of
%d)' % (currentkey, len(keys))
failedkeys.append(key)
except freenet.SegmentFailedError, err:
- print "Splitfile segment %d failed with %d of %d blocks" % \
- (err[0] + 1, err[1], err[2])
+ print color(err.getErrorMessage(), failedcolor), '(Key %d of
%d)' % (currentkey, len(keys))
failedkeys.append(key)
except (freenet.FormatError, freenet.URIError), err:
- print "Malformed key:", key
+ print color(err.getErrorMessage(), failedcolor), '(Key %d of
%d)' % (currentkey, len(keys))
failedkeys.append(key)
except freenet.KeyNotFoundError, err:
- print "The key %s was not found in the manifest!" % err[0]
+ print color(err.getErrorMessage(), failedcolor), '(Key %d of
%d)' % (currentkey, len(keys))
failedkeys.append(key)
- except freenet.TimeoutError:
- print 'Timed out!'
+ except freenet.TimeoutError, err:
+ print color(err.getErrorMessage(), failedcolor), '(Key %d of
%d)' % (currentkey, len(keys))
failedkeys.append(key)
- except freenet.ChecksumFailedError:
- print 'Checksum failed!'
+ except freenet.SplitFileError, err:
+ doc = (metadatalib.parse(err.getMetadata()))[0]
+ print color("Retrieved splitfile info, skipping file
download:", splitfilecolor), '(Key %d of %d)' % (currentkey, len(keys))
+ print ' Size:', _formatsize(int(doc['SplitFile.Size'], 16))
+ print ' Blocks:', int(doc['SplitFile.BlockCount'], 16)
+ print ' Checkblocks:', int(doc['SplitFile.CheckBlockCount'],
16)
+ if doc.has_key('Info.Format'):
+ print ' Format:', doc['Info.Format']
+ if doc.has_key('Info.Checksum'):
+ print ' Checksum:', doc['Info.Checksum']
+ if doc.has_key('Info.Description'):
+ print ' Description:', doc['Info.Description']
+ splitfilekeys.append(key)
+ except freenet.ChecksumFailedError, err:
+ print color(err.getErrorMessage(), failedcolor)
print 'This might be an error in YAFI, or it might be an ' \
'error in Freenet. Try re-downloading the file.'
failedkeys.append(key)
else:
successkeys.append(key)
- if filename is sys.stdout:
+ if options.raw:
+ _rawprint(data)
+ if options.verbose:
+ print color('Success!', successcolor), '(Key %d of
%d)' % (currentkey, len(keys))
+ elif filename is sys.stdout:
print data
if options.verbose:
- print 'Success!'
+ print color('Success!', successcolor), '(Key %d of
%d)' % (currentkey, len(keys))
else:
file = open(filename, 'w')
file.write(data)
file.close()
- print 'Success!'
+ print color('Success!', successcolor), '(Key %d of %d)' %
(currentkey, len(keys))
# Now do healing.
if len(data) > freenet.MAXSIZE and not options.raw and \
@@ -236,42 +280,62 @@
threads=options.hthreads,
percent=(options.healpercent / 100.0),
skipblocks=n.successblocks())
- except freenet.HealingError:
- print 'Healing failed!'
+ except freenet.HealingError, err:
+ print color(err.getErrorMessage(), failedcolor)
else:
- print 'Healing was successful!'
+ print color('Healing was successful!', successcolor)
finally:
statthread.kill()
if options.report:
if len(successkeys) > 0:
- print "Successfully retrieved keys:"
+ print color('Successfully retrieved %d keys:' % len(successkeys),
successcolor)
for key in successkeys:
print key
if len(failedkeys) > 0:
- print "Failed keys:"
+ print color('Failed %d keys:' % len(failedkeys), failedcolor)
for key in failedkeys:
print key
if len(skippedkeys) > 0:
- print "Skipped keys:"
+ print color('Skipped %d keys:' % len(skippedkeys), skippedcolor)
for key in skippedkeys:
print key
+ if len(splitfilekeys) > 0:
+ print color('Skipped %d splitfiles:' % len(splitfilekeys),
splitfilecolor)
+ for key in splitfilekeys:
+ print key
return 0
+#This needs work
+def _rawprint(data):
+ if data is not None and len(data) > 0:
+ docs = metadatalib.parse(data)
+ for doc in docs:
+ if doc.has_key('Name') and len(doc['Name']) > 0:
+ key = doc['Redirect.Target']
+ if key.startswith('freenet:'):
+ key = key[8:]
+ print key + '/' + doc['Name']
+
def put_command(options, args):
global globaldata
#default number of threads
if options.threads is None:
- options.threads = 15
+ options.threads = 10
if options.verbose is True:
e = threading.Event()
else:
e = None
- n = freenet.Node(options.host, options.port, statevent=e)
+ try:
+ n = freenet.Node(options.host, options.port, statevent=e)
+ except freenet.ConnectError, err:
+ print err.getErrorMessage()
+ return 1
+
statthread = StatThread(e, n, direction='Out')
statthread.start()
@@ -285,29 +349,29 @@
try:
f = open(filename)
except OSError:
- print "Failure reading file %s" % filename
+ print 'Failure reading file %s' % filename
failedfiles.append(filename)
continue
data = f.read()
f.close()
- print "Inserting %s..." % filename
+ print 'Inserting %s...' % filename
try:
key = n.put(data, htl=options.htl, threads=options.threads,
removelocal=True, filename=filename)
except freenet.RNFError:
- print 'Route not found!'
+ print color(err.getErrorMessage(), failedcolor)
failedfiles.append(filename)
except freenet.TimeoutError:
- print 'Timed out!'
+ print color(err.getErrorMessage(), failedcolor)
failedfiles.append(filename)
except (freenet.FormatError, freenet.URIError), err:
- print "Unknown error:", filename
+ print color(err.getErrorMessage(), failedcolor)
failedfiles.append(filename)
except freenet.KeyCollisionError, err:
- print "Key collision: %s as %s" % (filename, err[0])
- collisionfiles.append(err[0] + '/' +
+ print color(err.getErrorMessage(), failedcolor)
+ collisionfiles.append(err.getURI() + '/' +
filename[filename.rfind('/') + 1:])
else:
if key['URI'].startswith('CHK'):
@@ -322,22 +386,27 @@
if options.report:
if len(successfiles) > 0:
- print "Successfully inserted files:"
+ print color('Successfully inserted files:', successcolor)
for file in successfiles:
print file
if len(failedfiles) > 0:
- print "Failed files:"
+ print color('Failed files:', failedcolor)
for file in failedfiles:
print file
if len(collisionfiles) > 0:
- print "Collided files:"
+ print color('Collided files:', skippedcolor)
for file in collisionfiles:
print file
return 0
def info_command(options, args):
- f = fcp.FCP(options.host, options.port)
+ try:
+ f = fcp.FCP(options.host, options.port)
+ except freenet.ConnectError, err:
+ print err.getErrorMessage()
+ return 1
+
info = f.info()
hello = f.hello()
for k in hello.keys():
@@ -362,9 +431,10 @@
data = o.read()
o.close()
+ encoder = fec.FEC(fec.DEFAULT_FEC)
if len(data) > freenet.MAXSIZE:
- blocks, checkblocks = fec.encode(data)
- metadata = n.createmetadata(data, filename)
+ blocks, checkblocks = encoder.encode(data)
+ metadata = metadatalib.splitfilecreate(data, blocks=blocks,
checkblocks=checkblocks, filename=filename, f=f, encoder=encoder)
chk = f.genCHK(data='', metadata=metadata)
print "%s/%s" % (chk, os.path.basename(filename))
else:
@@ -374,7 +444,12 @@
return 0
def svk_command(options, args):
- f = fcp.FCP(options.host, options.port)
+ try:
+ f = fcp.FCP(options.host, options.port)
+ except freenet.ConnectError, err:
+ print err.getErrorMessage()
+ return 1
+
res = f.genSVK()
print 'Public Key: ', res['PublicKey']
print 'Private Key:', res['PrivateKey']
@@ -382,6 +457,73 @@
return 0
+def splitfileinfo_command(options, keys):
+ if options.infile is not None:
+ if options.infile == '-':
+ print "Reading keys from standard input..."
+ infile = sys.stdin
+ else:
+ print "Reading keys from %s..." % options.infile
+ try:
+ infile = open(options.infile)
+ except IOError:
+ print "Input file not found!"
+ return 2
+
+ for line in infile.readlines():
+ if line[:3] == 'CHK' or line[:3] == 'KSK' or line[:3] == 'SSK':
+ keys.append(line.strip())
+
+ #sanitize the keys
+ for i in range(len(keys)):
+ keys[i] = keys[i].strip('"')
+ keys[i] = urllib.unquote(keys[i])
+
+ try:
+ n = freenet.Node(options.host, options.port)
+ except freenet.ConnectError, err:
+ print err.getErrorMessage()
+ return 1
+
+ for key in keys:
+ print 'Fetching splitfile info for %s:' % key
+
+ try:
+ data = n.get(key, htl=options.htl,
+ splitfile=False)
+ except freenet.DNFError, err:
+ print color(err.getErrorMessage(), failedcolor)
+ except freenet.RNFError, err:
+ print color(err.getErrorMessage(), failedcolor)
+ except freenet.FECError, err:
+ print color(err.getErrorMessage(), failedcolor)
+ except freenet.SegmentFailedError, err:
+ print color(err.getErrorMessage(), failedcolor)
+ except (freenet.FormatError, freenet.URIError), err:
+ print color(err.getErrorMessage(), failedcolor)
+ except freenet.KeyNotFoundError, err:
+ print color(err.getErrorMessage(), failedcolor)
+ except freenet.TimeoutError, err:
+ print color(err.getErrorMessage(), failedcolor)
+ except freenet.SplitFileError, err:
+ doc = (metadatalib.parse(err.getMetadata()))[0]
+ print color("Successfully retrieved info:", successcolor)
+ print ' Size:', _formatsize(int(doc['SplitFile.Size'], 16))
+ print ' Blocks:', int(doc['SplitFile.BlockCount'], 16)
+ print ' Checkblocks:', int(doc['SplitFile.CheckBlockCount'], 16)
+ if doc.has_key('Info.Format'):
+ print ' Format:', doc['Info.Format']
+ if doc.has_key('Info.Checksum'):
+ print ' Checksum:', doc['Info.Checksum']
+ if doc.has_key('Info.Description'):
+ print ' Description:', doc['Info.Description']
+ except freenet.ChecksumFailedError, err:
+ print color(err.getErrorMessage(), failedcolor)
+ else:
+ print color('Not a splitfile!', failedcolor)
+
+ return 0
+
# God, this whole thing is a horrid hack.
# The GUI will be better, I promise.
class StatThread(threading.Thread):
@@ -426,20 +568,6 @@
self._transfers = []
def _display(self, status):
- def fmtstr(n):
- factor = 1000.0
- prefixes = ['', 'kilo', 'mega', 'giga', 'tera', 'peta', 'exa']
- unit = 'byte'
- c = factor
- for prefix in prefixes:
- if n < c:
- res = '%.1f' % (n / (c / factor))
- res += ' ' + prefix + unit
- if n > 1:
- res += 's'
- break
- c *= factor
- return res
def fmttransfers(transfers):
res = ''
lt = None
@@ -502,7 +630,7 @@
self._restarts = status['Transfers']['Restarts']
elif status['State'] == 'Splitfile':
if not self._splitfile:
- print "Splitfile: %s" % fmtstr(int(status['Size']))
+ print "Splitfile: %s" % _formatsize(int(status['Size']))
self._splitfile = True
elif status['Segment'][0] > self._segment:
print "Segment %d of %d" % (status['Segment'][0],
status['Segment'][1])
@@ -516,7 +644,7 @@
failedblocks = len(status['FailedBlocks'])
if doneblocks > self._doneblocks: #new blocks are finished
for n in range(doneblocks - self._doneblocks):
- print color("Got block %d" %
(status['DoneBlocks'][-(n + 1)] + 1), "green")
+ print color("Got block %d" %
(status['DoneBlocks'][-(n + 1)] + 1), successcolor)
print "Got %d of %d blocks." %
(len(status['DoneBlocks']), status['Blocks'][0])
self._doneblocks = doneblocks
elif failedblocks > self._failedblocks:
@@ -524,9 +652,9 @@
blockid = status['FailedBlocks'][-(n + 1)]
count = status['FailedBlocks'].count(blockid)
if count > 1:
- print color("DNF: block %d (%d times)" %
(blockid + 1, count), "red")
+ print color("DNF: block %d (%d times)" %
(blockid + 1, count), failedcolor)
else:
- print color("DNF: block %d (1 time)" %
(blockid + 1), "red")
+ print color("DNF: block %d (1 time)" %
(blockid + 1), failedcolor)
self._failedblocks = failedblocks
if not cmptransfers(self._transfers, status['Transfers'])
and not allstopped(status['Transfers']):
print "Transferring blocks",
fmttransfers(status['Transfers']) #map(lambda a: a + 1, status['ActiveBlocks'])
@@ -541,7 +669,7 @@
failedblocks = len(status['FailedBlocks'])
if doneblocks > self._doneblocks: #new blocks are finished
for n in range(doneblocks - self._doneblocks):
- print color("Inserted block %d." %
(status['DoneBlocks'][-(n + 1)] + 1), "green")
+ print color("Inserted block %d." %
(status['DoneBlocks'][-(n + 1)] + 1), successcolor)
print "Inserted %d of %d blocks." %
(len(status['DoneBlocks']), status['Blocks'])
self._doneblocks = doneblocks
if failedblocks > self._failedblocks:
@@ -549,9 +677,9 @@
blockid = status['FailedBlocks'][-(n + 1)]
count = status['FailedBlocks'].count(blockid)
if count > 1:
- print color("Block %d failed %d times" %
(blockid + 1, count), "red")
+ print color("Block %d failed %d times" %
(blockid + 1, count), failedcolor)
else:
- print color("Block %d failed 1 time" %
(blockid + 1), "red")
+ print color("Block %d failed 1 time" %
(blockid + 1), failedcolor)
self._failedblocks = failedblocks
if not cmptransfers(self._transfers, status['Transfers'])
and not allstopped(status['Transfers']):
print "Transferring blocks",
fmttransfers(status['Transfers']) #map(lambda a: a + 1, status['ActiveBlocks'])
@@ -585,7 +713,7 @@
self._restarts = status['Transfers']['Restarts']
elif status['State'] == 'Splitfile':
if not self._splitfile:
- print "Splitfile: %s, %d blocks" %
(fmtstr(int(status['Size'])), status['Blocks'])
+ print "Splitfile: %s, %d blocks" %
(_formatsize(int(status['Size'])), status['Blocks'])
print "Key is: %s" % status['Key']
self._splitfile = True
else:
@@ -593,7 +721,7 @@
failedblocks = len(status['FailedBlocks'])
if doneblocks > self._doneblocks: #new blocks are finished
for n in range(doneblocks - self._doneblocks):
- print color("Inserted block %d." %
(status['DoneBlocks'][-(n + 1)] + 1), "green")
+ print color("Inserted block %d." %
(status['DoneBlocks'][-(n + 1)] + 1), successcolor)
print "Finished %d of %d blocks." %
(len(status['DoneBlocks']), status['Blocks'])
self._doneblocks = doneblocks
if failedblocks > self._failedblocks:
@@ -601,9 +729,9 @@
blockid = status['FailedBlocks'][-(n + 1)]
count = status['FailedBlocks'].count(blockid)
if count > 1:
- print color("Block %d failed %d times." %
(blockid + 1, count), "red")
+ print color("Block %d failed %d times." %
(blockid + 1, count), failedcolor)
else:
- print color("Block %d failed 1 time." %
(blockid + 1), "red")
+ print color("Block %d failed 1 time." %
(blockid + 1), failedcolor)
self._failedblocks = failedblocks
if not cmptransfers(self._transfers, status['Transfers'])
and not allstopped(status['Transfers']):
print "Transferring blocks",
fmttransfers(status['Transfers']) #map(lambda a: a + 1, status['ActiveBlocks'])
@@ -618,31 +746,49 @@
# Following color stuff adapted from Andrei Kulakov
# http://silmarill.org/files/avkutil.py
colors = {
- "black" : "30",
- "red" : "31",
- "green" : "32",
- "brown" : "33",
- "blue" : "34",
- "purple" : "35",
- "cyan" : "36",
- "lgray" : "37",
- "gray" : "1;30",
- "lred" : "1;31",
- "lgreen" : "1;32",
- "yellow" : "1;33",
- "lblue" : "1;34",
- "pink" : "1;35",
- "lcyan" : "1;36",
- "white" : "1;37"
+ 'black' : '30',
+ 'red' : '31',
+ 'green' : '32',
+ 'brown' : '33',
+ 'blue' : '34',
+ 'purple' : '35',
+ 'cyan' : '36',
+ 'lgray' : '37',
+ 'gray' : '1;30',
+ 'lred' : '1;31',
+ 'lgreen' : '1;32',
+ 'yellow' : '1;33',
+ 'lblue' : '1;34',
+ 'pink' : '1;35',
+ 'lcyan' : '1;36',
+ 'white' : '1;37'
}
-def color(text, color):
+def color(text, color='white'):
opencol = "\033["
closecol = "m"
clear = opencol + "0" + closecol
fg = opencol + colors[color] + closecol
return "%s%s%s" % (fg, text, clear)
+def _formatsize(n):
+ factor = 1000.0
+ prefixes = ['', 'kilo', 'mega', 'giga', 'tera', 'peta', 'exa']
+ unit = 'byte'
+ c = factor
+ for prefix in prefixes:
+ if n < c:
+ res = '%.1f' % (n / (c / factor))
+ res += ' ' + prefix + unit
+ if n > 1:
+ res += 's'
+ break
+ c *= factor
+ return res
+
+# get our set of commands
+commands = map(lambda s: s[:-(len('_command'))], filter(lambda s:
s.endswith('_command'), dir()))
+
if __name__ == '__main__':
sys.exit(main(sys.argv))
