http://hg.viff.dk/viff/rev/6cd5ceb87542 changeset: 1184:6cd5ceb87542 user: Marcel Keller <[email protected]> date: Tue May 19 17:04:07 2009 +0200 summary: Added reentrant reactor.
Data now is sent faster and incoming data is processed faster. diffstat: 28 files changed, 186 insertions(+), 1 deletion(-) apps/aes.py | 2 + apps/aes_inversion.py | 2 + apps/beginner.py | 2 + apps/benchmark.py | 2 + apps/compare.py | 2 + apps/divide.py | 2 + apps/equality.py | 2 + apps/eval-poly.py | 2 + apps/gc-test.py | 2 + apps/int-bit-conversion.py | 2 + apps/millionaires.py | 2 + apps/multiply.py | 2 + apps/online-comparison-benchmark.py | 2 + apps/paillier.py | 2 + apps/prss-and-open.py | 2 + apps/seq-mul.py | 2 + apps/shamir-share-open.py | 2 + apps/share-open.py | 2 + apps/sort.py | 2 + apps/sum.py | 2 + apps/two-fields.py | 2 + apps/xor-all.py | 2 + viff/active.py | 11 +++++++ viff/passive.py | 11 +++++++ viff/reactor.py | 55 +++++++++++++++++++++++++++++++++++ viff/runtime.py | 49 ++++++++++++++++++++++++++++++- viff/test/__init__.py | 3 + viff/test/util.py | 14 ++++++++ diffs (528 lines): diff -r e89fb02c5e3d -r 6cd5ceb87542 apps/aes.py --- a/apps/aes.py Thu May 14 12:01:54 2009 +0200 +++ b/apps/aes.py Tue May 19 17:04:07 2009 +0200 @@ -23,6 +23,8 @@ import time from optparse import OptionParser +import viff.reactor +viff.reactor.install() from twisted.internet import reactor from viff.field import GF256 diff -r e89fb02c5e3d -r 6cd5ceb87542 apps/aes_inversion.py --- a/apps/aes_inversion.py Thu May 14 12:01:54 2009 +0200 +++ b/apps/aes_inversion.py Tue May 19 17:04:07 2009 +0200 @@ -24,6 +24,8 @@ import time from optparse import OptionParser +import viff.reactor +viff.reactor.install() from twisted.internet import reactor from viff.field import GF256 diff -r e89fb02c5e3d -r 6cd5ceb87542 apps/beginner.py --- a/apps/beginner.py Thu May 14 12:01:54 2009 +0200 +++ b/apps/beginner.py Tue May 19 17:04:07 2009 +0200 @@ -37,6 +37,8 @@ # Some useful imports. import sys +import viff.reactor +viff.reactor.install() from twisted.internet import reactor from viff.field import GF diff -r e89fb02c5e3d -r 6cd5ceb87542 apps/benchmark.py --- a/apps/benchmark.py Thu May 14 12:01:54 2009 +0200 +++ b/apps/benchmark.py Tue May 19 17:04:07 2009 +0200 @@ -60,6 +60,8 @@ import operator from pprint import pformat +import viff.reactor +viff.reactor.install() from twisted.internet import reactor from viff.field import GF, GF256, FakeGF diff -r e89fb02c5e3d -r 6cd5ceb87542 apps/compare.py --- a/apps/compare.py Thu May 14 12:01:54 2009 +0200 +++ b/apps/compare.py Tue May 19 17:04:07 2009 +0200 @@ -20,6 +20,8 @@ import sys import random +import viff.reactor +viff.reactor.install() from twisted.internet import reactor from twisted.internet.defer import gatherResults diff -r e89fb02c5e3d -r 6cd5ceb87542 apps/divide.py --- a/apps/divide.py Thu May 14 12:01:54 2009 +0200 +++ b/apps/divide.py Tue May 19 17:04:07 2009 +0200 @@ -32,6 +32,8 @@ """ from optparse import OptionParser +import viff.reactor +viff.reactor.install() from twisted.internet import reactor from viff.field import GF diff -r e89fb02c5e3d -r 6cd5ceb87542 apps/equality.py --- a/apps/equality.py Thu May 14 12:01:54 2009 +0200 +++ b/apps/equality.py Tue May 19 17:04:07 2009 +0200 @@ -31,6 +31,8 @@ """ from optparse import OptionParser +import viff.reactor +viff.reactor.install() from twisted.internet import reactor from viff.field import GF diff -r e89fb02c5e3d -r 6cd5ceb87542 apps/eval-poly.py --- a/apps/eval-poly.py Thu May 14 12:01:54 2009 +0200 +++ b/apps/eval-poly.py Tue May 19 17:04:07 2009 +0200 @@ -26,6 +26,8 @@ from time import time from optparse import OptionParser +import viff.reactor +viff.reactor.install() from twisted.internet import reactor from viff.field import GF diff -r e89fb02c5e3d -r 6cd5ceb87542 apps/gc-test.py --- a/apps/gc-test.py Thu May 14 12:01:54 2009 +0200 +++ b/apps/gc-test.py Tue May 19 17:04:07 2009 +0200 @@ -26,6 +26,8 @@ import sys from time import time +import viff.reactor +viff.reactor.install() from twisted.internet import reactor from viff.field import GF diff -r e89fb02c5e3d -r 6cd5ceb87542 apps/int-bit-conversion.py --- a/apps/int-bit-conversion.py Thu May 14 12:01:54 2009 +0200 +++ b/apps/int-bit-conversion.py Tue May 19 17:04:07 2009 +0200 @@ -19,6 +19,8 @@ import sys +import viff.reactor +viff.reactor.install() from twisted.internet import reactor from viff.field import GF, GF256 diff -r e89fb02c5e3d -r 6cd5ceb87542 apps/millionaires.py --- a/apps/millionaires.py Thu May 14 12:01:54 2009 +0200 +++ b/apps/millionaires.py Tue May 19 17:04:07 2009 +0200 @@ -32,6 +32,8 @@ # the example with '--help' for help with the command line options. from optparse import OptionParser +import viff.reactor +viff.reactor.install() from twisted.internet import reactor from viff.field import GF diff -r e89fb02c5e3d -r 6cd5ceb87542 apps/multiply.py --- a/apps/multiply.py Thu May 14 12:01:54 2009 +0200 +++ b/apps/multiply.py Tue May 19 17:04:07 2009 +0200 @@ -18,6 +18,8 @@ # License along with VIFF. If not, see <http://www.gnu.org/licenses/>. from optparse import OptionParser +import viff.reactor +viff.reactor.install() from twisted.internet import reactor from viff.field import GF diff -r e89fb02c5e3d -r 6cd5ceb87542 apps/online-comparison-benchmark.py --- a/apps/online-comparison-benchmark.py Thu May 14 12:01:54 2009 +0200 +++ b/apps/online-comparison-benchmark.py Tue May 19 17:04:07 2009 +0200 @@ -21,6 +21,8 @@ import signal from optparse import OptionParser +import viff.reactor +viff.reactor.install() from twisted.internet import reactor from twisted.internet.defer import DeferredList, gatherResults diff -r e89fb02c5e3d -r 6cd5ceb87542 apps/paillier.py --- a/apps/paillier.py Thu May 14 12:01:54 2009 +0200 +++ b/apps/paillier.py Tue May 19 17:04:07 2009 +0200 @@ -26,6 +26,8 @@ import sys +import viff.reactor +viff.reactor.install() from twisted.internet import reactor from viff.field import GF diff -r e89fb02c5e3d -r 6cd5ceb87542 apps/prss-and-open.py --- a/apps/prss-and-open.py Thu May 14 12:01:54 2009 +0200 +++ b/apps/prss-and-open.py Tue May 19 17:04:07 2009 +0200 @@ -19,6 +19,8 @@ import sys +import viff.reactor +viff.reactor.install() from twisted.internet import reactor from twisted.internet.defer import gatherResults diff -r e89fb02c5e3d -r 6cd5ceb87542 apps/seq-mul.py --- a/apps/seq-mul.py Thu May 14 12:01:54 2009 +0200 +++ b/apps/seq-mul.py Tue May 19 17:04:07 2009 +0200 @@ -18,6 +18,8 @@ # License along with VIFF. If not, see <http://www.gnu.org/licenses/>. from optparse import OptionParser +import viff.reactor +viff.reactor.install() from twisted.internet import reactor from viff.field import GF diff -r e89fb02c5e3d -r 6cd5ceb87542 apps/shamir-share-open.py --- a/apps/shamir-share-open.py Thu May 14 12:01:54 2009 +0200 +++ b/apps/shamir-share-open.py Tue May 19 17:04:07 2009 +0200 @@ -19,6 +19,8 @@ import sys +import viff.reactor +viff.reactor.install() from twisted.internet import reactor from viff.field import GF diff -r e89fb02c5e3d -r 6cd5ceb87542 apps/share-open.py --- a/apps/share-open.py Thu May 14 12:01:54 2009 +0200 +++ b/apps/share-open.py Tue May 19 17:04:07 2009 +0200 @@ -19,6 +19,8 @@ import sys +import viff.reactor +viff.reactor.install() from twisted.internet import reactor from viff.field import GF256 diff -r e89fb02c5e3d -r 6cd5ceb87542 apps/sort.py --- a/apps/sort.py Thu May 14 12:01:54 2009 +0200 +++ b/apps/sort.py Tue May 19 17:04:07 2009 +0200 @@ -51,6 +51,8 @@ from math import log, floor from optparse import OptionParser +import viff.reactor +viff.reactor.install() from twisted.internet import reactor from progressbar import ProgressBar, Percentage, Bar, ETA, ProgressBarWidget diff -r e89fb02c5e3d -r 6cd5ceb87542 apps/sum.py --- a/apps/sum.py Thu May 14 12:01:54 2009 +0200 +++ b/apps/sum.py Tue May 19 17:04:07 2009 +0200 @@ -18,6 +18,8 @@ # License along with VIFF. If not, see <http://www.gnu.org/licenses/>. from optparse import OptionParser +import viff.reactor +viff.reactor.install() from twisted.internet import reactor from viff.field import GF diff -r e89fb02c5e3d -r 6cd5ceb87542 apps/two-fields.py --- a/apps/two-fields.py Thu May 14 12:01:54 2009 +0200 +++ b/apps/two-fields.py Tue May 19 17:04:07 2009 +0200 @@ -19,6 +19,8 @@ import sys +import viff.reactor +viff.reactor.install() from twisted.internet import reactor from viff.field import GF diff -r e89fb02c5e3d -r 6cd5ceb87542 apps/xor-all.py --- a/apps/xor-all.py Thu May 14 12:01:54 2009 +0200 +++ b/apps/xor-all.py Tue May 19 17:04:07 2009 +0200 @@ -19,6 +19,8 @@ import sys +import viff.reactor +viff.reactor.install() from twisted.internet import reactor from viff.field import GF256 diff -r e89fb02c5e3d -r 6cd5ceb87542 viff/active.py --- a/viff/active.py Thu May 14 12:01:54 2009 +0200 +++ b/viff/active.py Tue May 19 17:04:07 2009 +0200 @@ -70,6 +70,9 @@ for protocol in self.protocols.itervalues(): protocol.sendData(pc, data_type, message) + # do actual communication + self.activate_reactor() + def echo_received(message, peer_id): # This is called when we receive an echo message. It # updates the echo count for the message and enters the @@ -133,6 +136,8 @@ d_send = Deferred().addCallback(send_received) self._expect_data(sender, SEND, d_send) + # do actual communication + self.activate_reactor() return result @@ -261,6 +266,9 @@ # first T shares. return rvec[:T] + # do actual communication + self.activate_reactor() + result = gather_shares(svec[T:]) self.schedule_callback(result, exchange) return result @@ -361,6 +369,9 @@ # first T shares. return (rvec1[:T], rvec2[:T]) + # do actual communication + self.activate_reactor() + result = gather_shares([gather_shares(svec1[T:]), gather_shares(svec2[T:])]) self.schedule_callback(result, exchange) return result diff -r e89fb02c5e3d -r 6cd5ceb87542 viff/passive.py --- a/viff/passive.py Thu May 14 12:01:54 2009 +0200 +++ b/viff/passive.py Tue May 19 17:04:07 2009 +0200 @@ -104,6 +104,10 @@ result = share.clone() self.schedule_callback(result, exchange) + + # do actual communication + self.activate_reactor() + if self.id in receivers: return result @@ -204,6 +208,10 @@ result = gather_shares([share_a, share_b]) result.addCallback(lambda (a, b): a * b) self.schedule_callback(result, share_recombine) + + # do actual communication + self.activate_reactor() + return result def pow(self, share, exponent): @@ -501,6 +509,9 @@ else: results.append(self._expect_share(peer_id, field)) + # do actual communication + self.activate_reactor() + # Unpack a singleton list. if len(results) == 1: return results[0] diff -r e89fb02c5e3d -r 6cd5ceb87542 viff/reactor.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/viff/reactor.py Tue May 19 17:04:07 2009 +0200 @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2009 VIFF Development Team. +# +# This file is part of VIFF, the Virtual Ideal Functionality Framework. +# +# VIFF is free software: you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License (LGPL) as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# VIFF is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General +# Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with VIFF. If not, see <http://www.gnu.org/licenses/>. + +"""VIFF reactor to have control over the scheduling.""" + +__docformat__ = "restructuredtext" + +from twisted.internet.selectreactor import SelectReactor + + +class ViffReactor(SelectReactor): + """VIFF reactor. + + The only difference to the SelectReactor is the loop call. + From there, doIteration() can be called recursively.""" + + def __init__(self): + SelectReactor.__init__(self) + self.loopCall = lambda: None + + def setLoopCall(self, f): + self.loopCall = f + + def doIteration(self, t): + # Do the same as in mainLoop() first. + self.runUntilCurrent() + t2 = self.timeout() + + if (t2 is not None): + t = min(t, self.running and t2) + + SelectReactor.doIteration(self, t) + self.loopCall() + +def install(): + """Use the VIFF reactor.""" + reactor = ViffReactor() + from twisted.internet.main import installReactor + installReactor(reactor) diff -r e89fb02c5e3d -r 6cd5ceb87542 viff/runtime.py --- a/viff/runtime.py Thu May 14 12:01:54 2009 +0200 +++ b/viff/runtime.py Tue May 19 17:04:07 2009 +0200 @@ -318,7 +318,10 @@ deferred = deq.popleft() if not deq: del self.waiting_deferreds[key] - deferred.callback(data) + + # Just queue, callbacks will be executed + # in process_deferred_queue(). + self.factory.runtime.queue_deferred(deferred, data) else: deq = self.incoming_data.setdefault(key, deque()) deq.append(data) @@ -533,6 +536,14 @@ # communicating with ourselves. self.add_player(player, None) + #: Queue of deferreds and data. + self.deferred_queue = [] + #: Counter for calls of activate_reactor(). + self.activation_counter = 0 + #: Record the recursion depth. + self.depth_counter = 0 + self.max_depth = 0 + def add_player(self, player, protocol): self.players[player.id] = player self.num_players = len(self.players) @@ -761,6 +772,39 @@ Python integer.""" raise NotImplemented("Override this abstract method in a subclass.") + def queue_deferred(self, deferred, data): + """Put deferred and data into the queue.""" + + self.deferred_queue.append((deferred, data)) + + def process_deferred_queue(self): + """Execute the callbacks of the deferreds in the queue.""" + + while(self.deferred_queue): + deferred, data = self.deferred_queue.pop(0) + deferred.callback(data) + + def activate_reactor(self): + """Activate the reactor to do actual communcation. + + This is where the recursion happens.""" + + self.activation_counter += 1 + + # setting the number to n makes the reactor called + # only every n-th time + if (self.activation_counter >= 2): + self.depth_counter += 1 + + if (self.depth_counter > self.max_depth): + # Record the maximal depth reached. + self.max_depth = self.depth_counter + + reactor.doIteration(0) + + self.depth_counter -= 1 + self.activation_counter = 0 + def make_runtime_class(runtime_class=None, mixins=None): """Creates a new runtime class with *runtime_class* as a base @@ -912,6 +956,9 @@ print "Will connect to %s" % player connect(player.host, player.port) + # Process the deferred queue after every reactor iteration. + reactor.setLoopCall(runtime.process_deferred_queue) + return result if __name__ == "__main__": diff -r e89fb02c5e3d -r 6cd5ceb87542 viff/test/__init__.py --- a/viff/test/__init__.py Thu May 14 12:01:54 2009 +0200 +++ b/viff/test/__init__.py Tue May 19 17:04:07 2009 +0200 @@ -14,3 +14,6 @@ # # You should have received a copy of the GNU Lesser General Public # License along with VIFF. If not, see <http://www.gnu.org/licenses/>. + +import viff.reactor +viff.reactor.install() diff -r e89fb02c5e3d -r 6cd5ceb87542 viff/test/util.py --- a/viff/test/util.py Thu May 14 12:01:54 2009 +0200 +++ b/viff/test/util.py Tue May 19 17:04:07 2009 +0200 @@ -19,6 +19,7 @@ from twisted.internet.defer import Deferred, gatherResults, maybeDeferred from twisted.trial.unittest import TestCase +from twisted.internet import reactor from viff.passive import PassiveRuntime from viff.runtime import Share, ShareExchanger, ShareExchangerFactory @@ -127,6 +128,7 @@ self.close_sentinels = [] self.runtimes = [] + self.real_runtimes = [] for id in reversed(range(1, self.num_players+1)): _, players = load_config(configs[id]) self.create_loopback_runtime(id, players) @@ -164,6 +166,18 @@ # the Runtime, since we want everybody to wait until all # runtimes are ready. self.runtimes.append(result) + self.real_runtimes.append(runtime) + self.i = 0 + + # This loop call should ensure the queues of the parties are + # processed in a more or less fair manner. This is necessary + # because we have only one reactor for all parties here. + def loop_call(): + for runtime in self.real_runtimes[self.i:] + self.real_runtimes[:self.i]: + runtime.process_deferred_queue() + self.i = (self.i + 1) % len(self.real_runtimes) + + reactor.setLoopCall(loop_call) for peer_id in players: if peer_id != id: _______________________________________________ viff-commits mailing list [email protected] http://lists.viff.dk/listinfo.cgi/viff-commits-viff.dk
