commit 76cc0d4989de0a777fa62f4de5b60f07e0701da7
Author: Arturo Filastò <art...@filasto.net>
Date:   Tue Jul 26 02:05:58 2016 +0200

    Use NGDeck also for the command line version of ooniprobe
    
    * Remove code that is now dead.
---
 bin/Makefile                           |  81 --------
 bin/ooniprobe                          |  24 ---
 bin/ooniprobe-dev                      |  10 -
 bin/test/ooni/__init__.py              |   1 -
 ooni/backend_client.py                 |  95 +--------
 ooni/deck.py                           | 356 +++++++++++++++++++++------------
 ooni/director.py                       |   2 -
 ooni/geoip.py                          |   6 +-
 ooni/reporter.py                       |   2 +-
 ooni/scripts/oonideckgen.py            |   2 +-
 ooni/scripts/ooniprobe.py              |  19 ++
 ooni/scripts/oonireport.py             |   2 +-
 ooni/templates/tcpt.py                 |   1 -
 ooni/tests/mocks.py                    |   2 +-
 ooni/tests/test_backend_client.py      |  16 --
 ooni/tests/test_deck.py                | 243 +++++++++++-----------
 ooni/ui/cli.py                         | 124 +++++-------
 ooni/ui/web/client/index.html          |   2 +-
 scripts/__init__.py                    |   0
 scripts/set_caps/Makefile              |  81 ++++++++
 scripts/set_caps/__init__.py           |   0
 scripts/set_caps/test/__init__.py      |   0
 scripts/set_caps/test/ooni/__init__.py |   1 +
 23 files changed, 517 insertions(+), 553 deletions(-)

diff --git a/bin/Makefile b/bin/Makefile
deleted file mode 100644
index bc80843..0000000
--- a/bin/Makefile
+++ /dev/null
@@ -1,81 +0,0 @@
-# Wrappers for running ooniprobe as a non-root user.
-#
-# Build-Depends: cython, pythonX.Y-dev, libcap2-bin
-# Depends: libpythonX.Y
-#
-# $ make && make check
-# $ sudo make install # after installing the rest of ooni-probe
-# $ make installcheck_unsafe # runs complete tests as non-root
-#
-# `make` builds a program that has file capabilities set on it. This is just
-# ./ooniprobe compiled into a C program using Cython, so that one can set
-# capabilities directly on the resulting binary. This way, we avoid the need
-# for a separate child python interpreter with its own capabilities. Another
-# advantage is that libpython.so (needed by the program) would be automatically
-# upgraded by the system package manager. The version of python is hard-coded
-# into the wrapper at build time; making this dynamic is possible, but much
-# more complex and not yet implemented.
-#
-# Execution may additionally be limited to a particular unix group by using
-# chgrp(1) and chmod(1) to 'o-x,g+x' after installation.
-#
-
-# GNU Makefile conventions, see 
https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
-prefix = /usr/local
-exec_prefix = $(prefix)
-bindir = $(exec_prefix)/bin
-
-INSTALL = install
-PYTHON = python
-PYTHON_CONFIG = python-config
-CYTHON = cython
-SETCAP = setcap
-
-INSTALL_PROGRAM = $(INSTALL)
-PY_CFLAGS = $(shell $(PYTHON_CONFIG) --cflags)
-PY_LDFLAGS = $(shell $(PYTHON_CONFIG) --ldflags)
-
-BUILDDIR := ./build
-SCRIPTDIR := .
-TESTDIR := ./test
-CAP_SCRIPT := ooniprobe
-CAP_NEEDED := cap_net_admin,cap_net_raw
-
-# Unfortunately cython --embed ignores the arguments in the shebang line
-# So we need to patch the generated code ourselves.
-CYTHON_PRE_MAIN = extern int Py_IgnoreEnvironmentFlag; \
-                  Py_IgnoreEnvironmentFlag++; \
-                  extern int Py_NoUserSiteDirectory; \
-                  Py_NoUserSiteDirectory++;
-
-all: $(BUILDDIR)/$(CAP_SCRIPT)
-
-$(BUILDDIR)/$(CAP_SCRIPT): $(BUILDDIR)/$(CAP_SCRIPT).c Makefile
-       $(CC) $(PY_CFLAGS) $(PY_LDFLAGS) "$<" -o "$@"
-
-$(BUILDDIR)/$(CAP_SCRIPT).c: $(SCRIPTDIR)/$(CAP_SCRIPT) Makefile
-       mkdir -p "$(BUILDDIR)"
-       $(CYTHON) "$<" --embed=CYTHON_MAIN_SENTINEL -Werror -Wextra -o "$@"
-       sed -i \
-         -e 's/\(.*CYTHON_MAIN_SENTINEL.*{\)/\1 $(CYTHON_PRE_MAIN)/g' \
-         -e '/CYTHON_MAIN_SENTINEL[^{]*$$/,/{/s/{/{ $(CYTHON_PRE_MAIN)/g' \
-         -e 's/CYTHON_MAIN_SENTINEL/main/g' "$@"
-
-check: $(BUILDDIR)/$(CAP_SCRIPT)
-       # test that setcapped binary ignores PYTHONPATH
-       BIN="$$(realpath "$<")" && cd "$(TESTDIR)" && PYTHONPATH=. $$BIN 
--version
-
-install: $(BUILDDIR)/$(CAP_SCRIPT)
-       mkdir -p "$(DESTDIR)$(bindir)"
-       $(INSTALL_PROGRAM) -t "$(DESTDIR)$(bindir)" "$(BUILDDIR)/$(CAP_SCRIPT)"
-       $(SETCAP) "$(CAP_NEEDED)"+eip "$(DESTDIR)$(bindir)/$(CAP_SCRIPT)"
-
-installcheck_unsafe: $(BUILDDIR)/$(CAP_SCRIPT)
-       # run a standard check. note that because of hardcoded paths (for 
security)
-       # this can only work after you've installed your development copy
-       "./$<" -i /usr/share/ooni/decks/complete.deck
-
-clean:
-       rm -rf "$(BUILDDIR)"
-
-.PHONY: clean all check install installcheck%
diff --git a/bin/ooniprobe b/bin/ooniprobe
deleted file mode 100755
index 0274899..0000000
--- a/bin/ooniprobe
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env python
-import sys
-
-from twisted.internet import reactor
-
-from ooni.ui.cli import runWithDaemonDirector, runWithDirector
-from ooni.ui.cli import setupGlobalOptions
-
-exit_code=0
-
-global_options = setupGlobalOptions(logging=True, start_tor=True,
-                                    check_incoherences=True)
-if global_options['queue']:
-    d = runWithDaemonDirector(global_options)
-else:
-    d = runWithDirector(global_options)
-@d.addBoth
-def cb(result):
-    global exit_code
-    if result is not None:
-        exit_code=1
-    reactor.stop()
-reactor.run()
-sys.exit(exit_code)
diff --git a/bin/ooniprobe-dev b/bin/ooniprobe-dev
deleted file mode 100755
index c44a8a6..0000000
--- a/bin/ooniprobe-dev
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/sh
-# Developer script for running ooniprobe directly from the repository.
-# We don't automatically add "$PWD" to PYTHONPATH as that is a security risk
-# when run as /usr/bin/ooniprobe on an end user's computer.
-ROOTDIR=$(cd $(dirname $(dirname $0)) && pwd -P)
-if [ $? -ne 0 ]; then
-    echo "$0: cannot determine toplevel directory" 1>&2
-    exit 1
-fi
-PYTHONPATH="$ROOTDIR" exec $ROOTDIR/bin/ooniprobe "$@"
diff --git a/bin/test/ooni/__init__.py b/bin/test/ooni/__init__.py
deleted file mode 100644
index f2cd353..0000000
--- a/bin/test/ooni/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-raise ValueError("test failed! wrapper did not ignore polluted PWD. either the 
wrapper is faulty, or ooni is still unpatched (Tor bug #13581)")
diff --git a/ooni/backend_client.py b/ooni/backend_client.py
index 506aba4..7721f6f 100644
--- a/ooni/backend_client.py
+++ b/ooni/backend_client.py
@@ -134,6 +134,8 @@ class OONIBClient(object):
         return finished
 
     def queryBackend(self, method, urn, query=None, retries=3):
+        log.debug("Querying backend {0}{1} with {2}".format(self.base_address,
+                                                         urn, query))
         bodyProducer = None
         if query:
             bodyProducer = StringProducer(json.dumps(query))
@@ -212,105 +214,12 @@ class CollectorClient(OONIBClient):
 
         return d
 
-    def getInput(self, input_hash):
-        from ooni.deck import InputFile
-
-        input_file = InputFile(input_hash)
-        if input_file.descriptorCached:
-            return defer.succeed(input_file)
-        else:
-            d = self.queryBackend('GET', '/input/' + input_hash)
-
-            @d.addCallback
-            def cb(descriptor):
-                input_file.load(descriptor)
-                input_file.save()
-                return input_file
-
-            @d.addErrback
-            def err(err):
-                log.err("Failed to get descriptor for input %s" % input_hash)
-                log.exception(err)
-
-            return d
-
-    def getInputList(self):
-        return self.queryBackend('GET', '/input')
-
-    def downloadInput(self, input_hash):
-        from ooni.deck import InputFile
-
-        input_file = InputFile(input_hash)
-
-        if input_file.fileCached:
-            return defer.succeed(input_file)
-        else:
-            d = self.download('/input/' + input_hash + '/file', 
input_file.cached_file)
-
-            @d.addCallback
-            def cb(res):
-                input_file.verify()
-                return input_file
-
-            @d.addErrback
-            def err(err):
-                log.err("Failed to download the input file %s" % input_hash)
-                log.exception(err)
-
-            return d
-
     def getInputPolicy(self):
         return self.queryBackend('GET', '/policy/input')
 
     def getNettestPolicy(self):
         return self.queryBackend('GET', '/policy/nettest')
 
-    def getDeckList(self):
-        return self.queryBackend('GET', '/deck')
-
-    def getDeck(self, deck_hash):
-        from ooni.deck import Deck
-
-        deck = Deck(deck_hash)
-        if deck.descriptorCached:
-            return defer.succeed(deck)
-        else:
-            d = self.queryBackend('GET', '/deck/' + deck_hash)
-
-            @d.addCallback
-            def cb(descriptor):
-                deck.load(descriptor)
-                deck.save()
-                return deck
-
-            @d.addErrback
-            def err(err):
-                log.err("Failed to get descriptor for deck %s" % deck_hash)
-                log.exception(err)
-
-            return d
-
-    def downloadDeck(self, deck_hash):
-        from ooni.deck import Deck
-
-        deck = Deck(deck_hash)
-        if deck.fileCached:
-            return defer.succeed(deck)
-        else:
-            d = self.download('/deck/' + deck_hash + '/file', deck.cached_file)
-
-            @d.addCallback
-            def cb(res):
-                deck.verify()
-                return deck
-
-            @d.addErrback
-            def err(err):
-                log.err("Failed to download the deck %s" % deck_hash)
-                log.exception(err)
-
-            return d
-
     def createReport(self, test_details):
         request = {
             'software_name': test_details['software_name'],
diff --git a/ooni/deck.py b/ooni/deck.py
index 1746d26..eacd256 100644
--- a/ooni/deck.py
+++ b/ooni/deck.py
@@ -24,59 +24,6 @@ from ooni.utils import log
 
 from ooni.results import generate_summary
 
-class InputFile(object):
-    def __init__(self, input_hash, base_path=config.inputs_directory):
-        self.id = input_hash
-        cache_path = os.path.join(os.path.abspath(base_path), input_hash)
-        self.cached_file = cache_path
-        self.cached_descriptor = cache_path + '.desc'
-
-    @property
-    def descriptorCached(self):
-        if os.path.exists(self.cached_descriptor):
-            with open(self.cached_descriptor) as f:
-                descriptor = json.load(f)
-                self.load(descriptor)
-            return True
-        return False
-
-    @property
-    def fileCached(self):
-        if os.path.exists(self.cached_file):
-            try:
-                self.verify()
-            except AssertionError:
-                log.err("The input %s failed validation."
-                        "Going to consider it not cached." % self.id)
-                return False
-            return True
-        return False
-
-    def save(self):
-        with open(self.cached_descriptor, 'w+') as f:
-            json.dump({
-                'name': self.name,
-                'id': self.id,
-                'version': self.version,
-                'author': self.author,
-                'date': self.date,
-                'description': self.description
-            }, f)
-
-    def load(self, descriptor):
-        self.name = descriptor['name']
-        self.version = descriptor['version']
-        self.author = descriptor['author']
-        self.date = descriptor['date']
-        self.description = descriptor['description']
-
-    def verify(self):
-        digest = os.path.basename(self.cached_file)
-        with open(self.cached_file) as f:
-            file_hash = sha256(f.read())
-            assert file_hash.hexdigest() == digest
-
-
 def nettest_to_path(path, allow_arbitrary_paths=False):
     """
     Takes as input either a path or a nettest name.
@@ -136,6 +83,7 @@ def get_preferred_bouncer():
     else:
         return BouncerClient(bouncer_address)
 
+<<<<<<< d0fb4f37530aeb6b69fbc2985019464f8ec10312
 class Deck(InputFile):
     # this exists so we can mock it out in unittests
     _BouncerClient = BouncerClient
@@ -274,6 +222,7 @@ class Deck(InputFile):
                 i['test_options'][i['key']] = input_file.cached_file
 
 
+@defer.inlineCallbacks
 def lookup_collector_and_test_helpers(net_test_loaders,
                                       bouncer,
                                       preferred_backend,
@@ -329,19 +278,21 @@ def lookup_collector_and_test_helpers(net_test_loaders,
                 net_test_loader.testName)
 
         collector, test_helpers = \
-            find_collector_and_test_helpers(test_name=net_test_loader.testName,
-                                            
test_version=net_test_loader.testVersion,
-                                            
input_files=net_test_loader.inputFiles)
+            find_collector_and_test_helpers(
+                test_name=net_test_loader.testName,
+                test_version=net_test_loader.testVersion,
+                input_files=net_test_loader.inputFiles
+            )
 
         for option, name in net_test_loader.missingTestHelpers:
             test_helper_address_or_settings = test_helpers[name]
             net_test_loader.localOptions[option] = 
test_helper_address_or_settings
             net_test_loader.testHelpers[option] = 
test_helper_address_or_settings
 
-        if not net_test_loader.collector:
+        if not net_test_loader.collector and not no_collector:
+            log.debug("Using collector {0}".format(collector))
             net_test_loader.collector = collector
 
-
 @defer.inlineCallbacks
 def get_reachable_test_helpers_and_collectors(net_tests, preferred_backend):
     for net_test in net_tests:
@@ -579,10 +530,29 @@ def options_to_args(options, prepath=None):
             continue
         if k == "file":
             v = resolve_file_path(v, prepath)
-        args.append('--'+k)
+        if v == False or v == 0:
+            continue
+        if (len(k)) == 1:
+            args.append('-'+k)
+        else:
+            args.append('--'+k)
+        if isinstance(v, bool) or isinstance(v, int):
+            continue
         args.append(v)
     return args
 
+def normalize_options(options):
+    """
+    Takes some options that have a mixture of - and _ and returns the
+    equivalent options with only '_'.
+    """
+    normalized_opts = {}
+    for k, v in options.items():
+        normalized_key = k.replace('-', '_')
+        assert normalized_key not in normalized_opts, "The key {0} cannot be 
normalized".format(k)
+        normalized_opts[normalized_key] = v
+    return normalized_opts
+
 class UnknownTaskKey(Exception):
     pass
 
@@ -593,8 +563,14 @@ class DeckTask(object):
     _metadata_keys = ["name"]
     _supported_tasks = ["ooni"]
 
-    def __init__(self, data, parent_metadata={}, cwd=None):
-        self.parent_metadata = parent_metadata
+    def __init__(self, data,
+                 parent_metadata={},
+                 global_options={},
+                 cwd=None,
+                 arbitrary_paths=False):
+
+        self.parent_metadata = normalize_options(parent_metadata)
+        self.global_options = global_options
         self.cwd = cwd
         self.data = deepcopy(data)
 
@@ -605,10 +581,16 @@ class DeckTask(object):
         self.requires_tor = False
         self.requires_bouncer = False
 
+        # If this is set to true a deck can specify any path. It should only
+        #  be run against trusted decks or when you create a deck
+        # programmaticaly to a run test specified from the command line.
+        self._arbitrary_paths = arbitrary_paths
+
         self.ooni = {
             'bouncer_client': None,
             'test_details': {}
         }
+        self.output_path = None
 
         self._load(data)
 
@@ -619,7 +601,8 @@ class DeckTask(object):
                 raise MissingTaskDataKey(required_key)
 
         # This raises e.NetTestNotFound, we let it go onto the caller
-        nettest_path = nettest_to_path(task_data.pop("test_name"))
+        nettest_path = nettest_to_path(task_data.pop("test_name"),
+                                       self._arbitrary_paths)
 
         try:
             annotations = task_data.pop('annotations')
@@ -631,8 +614,16 @@ class DeckTask(object):
         except KeyError:
             collector_address = self.parent_metadata.get('collector', None)
 
+        try:
+            self.output_path = task_data.pop('reportfile')
+        except KeyError:
+            self.output_path = self.global_options.get('reportfile', None)
+
+        if task_data.get('no-collector', False):
+            collector_address = None
+
         net_test_loader = NetTestLoader(
-            options_to_args(task_data),
+            options_to_args(task_data, self.cwd),
             annotations=annotations,
             test_file=nettest_path
         )
@@ -658,11 +649,14 @@ class DeckTask(object):
             self.requires_bouncer = True
 
         self.ooni['net_test_loader'] = net_test_loader
-        # Need to ensure that this is called only once we have looked up the
-        #  probe IP address and have geoip data.
-        self.ooni['test_details'] = net_test_loader.getTestDetails()
+
+    def _setup_ooni(self):
+        self.ooni['test_details'] = 
self.ooni['net_test_loader'].getTestDetails()
         self.id = generate_filename(self.ooni['test_details'])
 
+    def setup(self):
+        getattr(self, "_setup_"+self.type)()
+
     def _load(self, data):
         for key in self._metadata_keys:
             try:
@@ -678,11 +672,81 @@ class DeckTask(object):
 
         assert len(data) == 0
 
+class NotAnOption(Exception):
+    pass
+
+def subargs_to_options(subargs):
+    options = {}
+
+    def parse_option_name(arg):
+        if arg.startswith("--"):
+            return arg[2:]
+        elif arg.startswith("-"):
+            return arg[1:]
+        raise NotAnOption
+
+    subargs = iter(reversed(subargs))
+    for subarg in subargs:
+        try:
+            value = subarg
+            name = parse_option_name(subarg)
+            options[name] = True
+        except NotAnOption:
+            try:
+                name = parse_option_name(subargs.next())
+                options[name] = value
+            except StopIteration:
+                break
+
+    return options
+
+def convert_legacy_deck(deck_data):
+    """
+    I take a legacy deck list and convert it to the new deck format.
+
+    :param deck_data: in the legacy format
+    :return: deck_data in the new format
+    """
+    assert isinstance(deck_data, list), "Legacy decks are lists"
+    new_deck_data = {}
+    new_deck_data["name"] = "Legacy deck"
+    new_deck_data["description"] = "This is a legacy deck converted to the " \
+                                   "new format"
+    new_deck_data["bouncer"] = None
+    new_deck_data["tasks"] = []
+    for deck_item in deck_data:
+        deck_task = {"ooni": {}}
+
+        options = deck_item["options"]
+        deck_task["ooni"]["test_name"] = options.pop("test_file")
+        deck_task["ooni"]["annotations"] = options.pop("annotations", {})
+        deck_task["ooni"]["collector"] = options.pop("collector", None)
+
+        # XXX here we end up picking only the last not none bouncer_address
+        bouncer_address = options.pop("bouncer", None)
+        if bouncer_address is not None:
+            new_deck_data["bouncer"] = bouncer_address
+
+        subargs = options.pop("subargs", [])
+        for name, value in subargs_to_options(subargs).items():
+            deck_task["ooni"][name] = value
+
+        for name, value in options.items():
+            deck_task["ooni"][name] = value
+
+        new_deck_data["tasks"].append(deck_task)
+
+    return new_deck_data
+
 class NGDeck(object):
-    def __init__(self, deck_data=None,
-                 deck_path=None, no_collector=False):
+    def __init__(self,
+                 deck_data=None,
+                 deck_path=None,
+                 global_options={},
+                 no_collector=False,
+                 arbitrary_paths=False):
         # Used to resolve relative paths inside of decks.
-        self.deck_directory = None
+        self.deck_directory = os.getcwd()
         self.requires_tor = False
         self.no_collector = no_collector
         self.name = ""
@@ -690,8 +754,12 @@ class NGDeck(object):
         self.schedule = None
 
         self.metadata = {}
+        self.global_options = normalize_options(global_options)
         self.bouncer = None
 
+        self._arbitrary_paths = arbitrary_paths
+        self._is_setup = False
+
         self._measurement_path = FilePath(config.measurements_directory)
         self._tasks = []
         self.task_ids = []
@@ -701,35 +769,24 @@ class NGDeck(object):
         elif deck_data is not None:
             self.load(deck_data)
 
-    def open(self, deck_path):
+    def open(self, deck_path, global_options=None):
         with open(deck_path) as fh:
             deck_data = yaml.safe_load(fh)
-        self.load(deck_data)
+        self.deck_directory = os.path.abspath(os.path.dirname(deck_path))
+        self.load(deck_data, global_options)
 
-    def write(self, fh):
-        """
-        Writes a properly formatted deck to the supplied file handle.
-        :param fh: an open file handle
-        :return:
-        """
-        deck_data = {
-            "name": self.name,
-            "description": self.description,
-            "tasks": [task.data for task in self._tasks]
-        }
-        if self.schedule is not None:
-            deck_data["schedule"] = self.schedule
-        for key, value in self.metadata.items():
-            deck_data[key] = value
+    def load(self, deck_data, global_options=None):
+        if global_options is not None:
+            self.global_options = global_options
 
-        fh.write("---\n")
-        yaml.safe_dump(deck_data, fh, default_flow_style=False)
+        if isinstance(deck_data, list):
+            deck_data = convert_legacy_deck(deck_data)
 
-    def load(self, deck_data):
         self.name = deck_data.pop("name", "Un-named Deck")
         self.description = deck_data.pop("description", "No description")
 
-        bouncer_address = deck_data.pop("bouncer", None)
+        bouncer_address = self.global_options.get('bouncer',
+                                                  deck_data.pop("bouncer", 
None))
         if bouncer_address is None:
             self.bouncer = get_preferred_bouncer()
         elif isinstance(bouncer_address, dict):
@@ -743,8 +800,17 @@ class NGDeck(object):
         for key, metadata in deck_data.items():
             self.metadata[key] = metadata
 
+        # We override the task metadata with the global options if present
+        self.metadata.update(self.global_options)
+
         for task_data in tasks_data:
-            deck_task = DeckTask(task_data, self.metadata, self.deck_directory)
+            deck_task = DeckTask(
+                data=task_data,
+                parent_metadata=self.metadata,
+                global_options=self.global_options,
+                cwd=self.deck_directory,
+                arbitrary_paths=self._arbitrary_paths
+            )
             if deck_task.requires_tor:
                 self.requires_tor = True
             if (deck_task.requires_bouncer and
@@ -753,6 +819,32 @@ class NGDeck(object):
             self._tasks.append(deck_task)
             self.task_ids.append(deck_task.id)
 
+        if self.metadata.get('no_collector', False):
+            self.no_collector = True
+
+    @property
+    def tasks(self):
+        return self._tasks
+
+    def write(self, fh):
+        """
+        Writes a properly formatted deck to the supplied file handle.
+        :param fh: an open file handle
+        :return:
+        """
+        deck_data = {
+            "name": self.name,
+            "description": self.description,
+            "tasks": [task.data for task in self._tasks]
+        }
+        if self.schedule is not None:
+            deck_data["schedule"] = self.schedule
+        for key, value in self.metadata.items():
+            deck_data[key] = value
+
+        fh.write("---\n")
+        yaml.safe_dump(deck_data, fh, default_flow_style=False)
+
     @defer.inlineCallbacks
     def query_bouncer(self):
         preferred_backend = config.advanced.get(
@@ -772,60 +864,74 @@ class NGDeck(object):
             preferred_backend,
             self.no_collector
         )
-
-    def _measurement_completed(self, result, measurement_id):
-        log.msg("{0}".format(result))
-        measurement_dir = self._measurement_path.child(measurement_id)
-        measurement_dir.child("measurements.njson.progress").moveTo(
-            measurement_dir.child("measurements.njson")
-        )
-        generate_summary(
-            measurement_dir.child("measurements.njson").path,
-            measurement_dir.child("summary.json").path
-        )
-        measurement_dir.child("running.pid").remove()
-
-    def _measurement_failed(self, failure, measurement_id):
-        measurement_dir = self._measurement_path.child(measurement_id)
-        measurement_dir.child("running.pid").remove()
-        # XXX do we also want to delete measurements.njson.progress?
+        defer.returnValue(net_test_loaders)
+
+    def _measurement_completed(self, result, task):
+        if not task.output_path:
+            measurement_id = task.id
+            measurement_dir = self._measurement_path.child(measurement_id)
+            measurement_dir.child("measurements.njson.progress").moveTo(
+                measurement_dir.child("measurements.njson")
+            )
+            generate_summary(
+                measurement_dir.child("measurements.njson").path,
+                measurement_dir.child("summary.json").path
+            )
+            measurement_dir.child("running.pid").remove()
+
+    def _measurement_failed(self, failure, task):
+        if not task.output_path:
+            # XXX do we also want to delete measurements.njson.progress?
+            measurement_id = task.id
+            measurement_dir = self._measurement_path.child(measurement_id)
+            measurement_dir.child("running.pid").remove()
         return failure
 
     def _run_ooni_task(self, task, director):
         net_test_loader = task.ooni["net_test_loader"]
         test_details = task.ooni["test_details"]
-        measurement_id = task.id
 
-        measurement_dir = self._measurement_path.child(measurement_id)
-        measurement_dir.createDirectory()
+        report_filename = task.output_path
+        if not task.output_path:
+            measurement_id = task.id
 
-        report_filename = 
measurement_dir.child("measurements.njson.progress").path
-        pid_file = measurement_dir.child("running.pid")
+            measurement_dir = self._measurement_path.child(measurement_id)
+            measurement_dir.createDirectory()
 
-        with pid_file.open('w') as out_file:
-            out_file.write("{0}".format(os.getpid()))
+            report_filename = 
measurement_dir.child("measurements.njson.progress").path
+            pid_file = measurement_dir.child("running.pid")
+
+            with pid_file.open('w') as out_file:
+                out_file.write("{0}".format(os.getpid()))
 
         d = director.start_net_test_loader(
             net_test_loader,
             report_filename,
+            collector_client=net_test_loader.collector,
             test_details=test_details
         )
-        d.addCallback(self._measurement_completed, measurement_id)
-        d.addErrback(self._measurement_failed, measurement_id)
+        d.addCallback(self._measurement_completed, task)
+        d.addErrback(self._measurement_failed, task)
         return d
 
+    def setup(self):
+        """
+        This method needs to be called before you are able to run a deck.
+        """
+        for task in self._tasks:
+            task.setup()
+        self._is_setup = True
+
     @defer.inlineCallbacks
     def run(self, director):
-        tasks = []
-        preferred_backend = config.advanced.get("preferred_backend", "onion")
+        assert self._is_setup, "You must call setup() before you can run a " \
+                               "deck"
+        if self.requires_tor:
+            yield director.start_tor()
         yield self.query_bouncer()
         for task in self._tasks:
-            if task.requires_tor:
-                yield director.start_tor()
-            elif task.requires_bouncer and preferred_backend == "onion":
-                yield director.start_tor()
             if task.type == "ooni":
-                tasks.append(self._run_ooni_task(task, director))
-        defer.returnValue(tasks)
+                yield self._run_ooni_task(task, director)
+        self._is_setup = False
 
 input_store = InputStore()
diff --git a/ooni/director.py b/ooni/director.py
index 793975e..c239601 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -424,9 +424,7 @@ class Director(object):
             log.debug("Setting SOCKS port as %s" % tor_config.SocksPort)
             try:
                 yield start_tor(tor_config)
-                log.err("Calling tor callback")
                 self._tor_starting.callback(self._tor_state)
-                log.err("called")
             except Exception as exc:
                 log.err("Failed to start tor")
                 log.exc(exc)
diff --git a/ooni/geoip.py b/ooni/geoip.py
index 28e0e1e..f118268 100644
--- a/ooni/geoip.py
+++ b/ooni/geoip.py
@@ -1,6 +1,7 @@
 from __future__ import absolute_import
 import re
 import os
+import json
 import random
 
 from hashlib import sha256
@@ -137,11 +138,12 @@ class UbuntuGeoIP(HTTPGeoIPLookupper):
         return probe_ip
 
 class DuckDuckGoGeoIP(HTTPGeoIPLookupper):
-    url = "https://duckduckgo.com/?q=ip&ia=answer";
+    url = "https://api.duckduckgo.com/?q=ip&format=json";
 
     def parseResponse(self, response_body):
+        j = json.loads(response_body)
         regexp = "Your IP address is (.*) in "
-        probe_ip = re.search(regexp, response_body).group(1)
+        probe_ip = re.search(regexp, j['Answer']).group(1)
         return probe_ip
 
 class ProbeIP(object):
diff --git a/ooni/reporter.py b/ooni/reporter.py
index f07b3cf..e2f155f 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -534,7 +534,7 @@ class Report(object):
             collector:
                 The address of the oonib collector for this report.
 
-            no_yamloo:
+            no_njson:
                 If we should disable reporting to disk.
         """
         self.test_details = test_details
diff --git a/ooni/scripts/oonideckgen.py b/ooni/scripts/oonideckgen.py
index fa675f9..b980a2c 100644
--- a/ooni/scripts/oonideckgen.py
+++ b/ooni/scripts/oonideckgen.py
@@ -92,7 +92,7 @@ def get_user_country_code():
 
 
 @defer.inlineCallbacks
-def oonideckgen():
+def oonideckgen(reactor):
     options = Options()
     try:
         options.parseOptions()
diff --git a/ooni/scripts/ooniprobe.py b/ooni/scripts/ooniprobe.py
new file mode 100644
index 0000000..24493da
--- /dev/null
+++ b/ooni/scripts/ooniprobe.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+from twisted.internet import task
+
+def ooniprobe(reactor):
+    from ooni.ui.cli import runWithDaemonDirector, runWithDirector
+    from ooni.ui.cli import setupGlobalOptions
+
+    global_options = setupGlobalOptions(logging=True, start_tor=True,
+                                        check_incoherences=True)
+    if global_options['queue']:
+        return runWithDaemonDirector(global_options)
+    else:
+        return runWithDirector(global_options)
+
+def run():
+    task.react(ooniprobe)
+
+if __name__ == "__main__":
+    run()
diff --git a/ooni/scripts/oonireport.py b/ooni/scripts/oonireport.py
index 6facadf..d75c925 100644
--- a/ooni/scripts/oonireport.py
+++ b/ooni/scripts/oonireport.py
@@ -202,7 +202,7 @@ def tor_check():
         sys.exit(1)
 
 
-def oonireport(args=sys.argv[1:]):
+def oonireport(reactor, args=sys.argv[1:]):
     options = Options()
     try:
         options.parseOptions(args)
diff --git a/ooni/templates/tcpt.py b/ooni/templates/tcpt.py
index 5e1d21a..bb080e6 100644
--- a/ooni/templates/tcpt.py
+++ b/ooni/templates/tcpt.py
@@ -96,4 +96,3 @@ class TCPTest(NetTestCase):
         d2.addCallback(connected)
         d2.addErrback(errback)
         return d1
-
diff --git a/ooni/tests/mocks.py b/ooni/tests/mocks.py
index c15beec..d623e0b 100644
--- a/ooni/tests/mocks.py
+++ b/ooni/tests/mocks.py
@@ -191,7 +191,7 @@ class MockTaskManager(TaskManager):
 
 class MockBouncerClient(object):
     def __init__(self, *args, **kw):
-        pass
+        self.backend_type = "onion"
 
     def lookupTestHelpers(self, required_test_helpers):
         ret = {
diff --git a/ooni/tests/test_backend_client.py 
b/ooni/tests/test_backend_client.py
index 212abb6..fab1fa2 100644
--- a/ooni/tests/test_backend_client.py
+++ b/ooni/tests/test_backend_client.py
@@ -60,22 +60,6 @@ class TestEnd2EndBackendClient(ConfigTestCase):
     def test_download_input(self):
         yield self.collector_client.downloadInput(input_id)
 
-    @defer.inlineCallbacks
-    def test_get_deck_list(self):
-        deck_list = yield self.collector_client.getDeckList()
-        self.assertTrue(isinstance(deck_list, list))
-
-    @defer.inlineCallbacks
-    def test_get_deck_descriptor(self):
-        deck_descriptor = yield self.collector_client.getDeck(deck_id)
-        for key in ['name', 'description',
-                    'version', 'author', 'date', 'id']:
-            self.assertTrue(hasattr(deck_descriptor, key))
-
-    @defer.inlineCallbacks
-    def test_download_deck(self):
-        yield self.collector_client.downloadDeck(deck_id)
-
     def test_lookup_invalid_helpers(self):
         bouncer_client = BouncerClient('http://127.0.0.1:8888')
         return self.failUnlessFailure(
diff --git a/ooni/tests/test_deck.py b/ooni/tests/test_deck.py
index 455f4e1..3d07959 100644
--- a/ooni/tests/test_deck.py
+++ b/ooni/tests/test_deck.py
@@ -1,15 +1,20 @@
 import os
 
+from StringIO import StringIO
 from copy import deepcopy
 
+import yaml
+
+from mock import patch, MagicMock
+
 from twisted.internet import defer
 from twisted.trial import unittest
 
 from hashlib import sha256
 from ooni import errors
 from ooni.deck import input_store, lookup_collector_and_test_helpers
-from ooni.nettest import NetTestLoader
-from ooni.deck import InputFile, Deck, nettest_to_path, DeckTask, NGDeck
+from ooni.deck import nettest_to_path, NGDeck
+from ooni.deck import convert_legacy_deck
 from ooni.tests.bases import ConfigTestCase
 from ooni.tests.mocks import MockBouncerClient, MockCollectorClient
 
@@ -80,45 +85,6 @@ class BaseTestCase(unittest.TestCase):
 """
         super(BaseTestCase, self).setUp()
 
-
-class TestInputFile(BaseTestCase):
-    def tearDown(self):
-        if self.filename != "":
-            os.remove(self.filename)
-
-    def test_file_cached(self):
-        self.filename = file_hash = sha256(self.dummy_deck_content).hexdigest()
-        input_file = InputFile(file_hash, base_path='.')
-        with open(file_hash, 'w+') as f:
-            f.write(self.dummy_deck_content)
-        assert input_file.fileCached
-
-    def test_file_invalid_hash(self):
-        self.filename = invalid_hash = 'a' * 64
-        with open(invalid_hash, 'w+') as f:
-            f.write("b" * 100)
-        input_file = InputFile(invalid_hash, base_path='.')
-        self.assertRaises(AssertionError, input_file.verify)
-
-    def test_save_descriptor(self):
-        descriptor = {
-            'name': 'spam',
-            'id': 'spam',
-            'version': 'spam',
-            'author': 'spam',
-            'date': 'spam',
-            'description': 'spam'
-        }
-        file_id = 'a' * 64
-        self.filename = file_id + '.desc'
-        input_file = InputFile(file_id, base_path='.')
-        input_file.load(descriptor)
-        input_file.save()
-        assert os.path.isfile(self.filename)
-
-        assert input_file.descriptorCached
-
-
 class TestDeck(BaseTestCase, ConfigTestCase):
     def setUp(self):
         super(TestDeck, self).setUp()
@@ -137,60 +103,47 @@ class TestDeck(BaseTestCase, ConfigTestCase):
         super(TestDeck, self).tearDown()
 
     def test_open_deck(self):
-        deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS,
-                    decks_directory=".")
-        deck.loadDeck(self.deck_file)
-        assert len(deck.netTestLoaders) == 1
+        deck = NGDeck()
+        deck.open(self.deck_file)
+        assert len(deck.tasks.ooni['net_test_loaders']) == 1
 
     def test_load_deck_with_global_options(self):
         global_options = {
             "annotations": {"spam": "ham"},
             "collector": "httpo://thirteenchars123.onion"
         }
-        deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS,
-                    decks_directory=".")
-        deck.loadDeck(self.deck_file,
-                      global_options=global_options)
+        deck = NGDeck(global_options=global_options)
+        deck.open(self.deck_file)
         self.assertEqual(
-            deck.netTestLoaders[0].annotations,
+            deck.tasks.ooni['net_test_loaders'][0].annotations,
             global_options['annotations']
         )
         self.assertEqual(
-            deck.netTestLoaders[0].collector.base_address,
+            deck.tasks.ooni['net_test_loaders'][0].collector.base_address,
             global_options['collector'].replace("httpo://", "http://";)
         )
 
-    def test_save_deck_descriptor(self):
-        deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS,
-                    decks_directory=".")
-        deck.loadDeck(self.deck_file)
-        deck.load({'name': 'spam',
-                   'id': 'spam',
-                   'version': 'spam',
-                   'author': 'spam',
-                   'date': 'spam',
-                   'description': 'spam'
-                   })
-        deck.save()
-        self.filename = self.deck_file + ".desc"
-        deck.verify()
-
+    @patch('ooni.deck.BouncerClient', MockBouncerClient)
+    @patch('ooni.deck.CollectorClient', MockCollectorClient)
     @defer.inlineCallbacks
     def test_lookup_test_helpers_and_collector(self):
-        deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS,
-                    decks_directory=".")
+        deck = NGDeck()
         deck.bouncer = MockBouncerClient(FAKE_BOUNCER_ADDRESS)
-        deck._BouncerClient = MockBouncerClient
-        deck._CollectorClient = MockCollectorClient
-        deck.loadDeck(self.deck_file)
+        deck.open(self.deck_file)
 
-        self.assertEqual(len(deck.netTestLoaders[0].missingTestHelpers), 1)
+        self.assertEqual(
+            len(deck.tasks.ooni['net_test_loaders'][0].missingTestHelpers), 1)
 
-        yield lookup_collector_and_test_helpers(deck.preferred_backend,
-                                                deck.netTestLoaders)
+        yield lookup_collector_and_test_helpers(
+            net_test_loaders=deck.netTestLoaders,
+            preferred_backend=deck.preferred_backend,
+            bouncer=deck.bouncer
+        )
 
-        self.assertEqual(deck.netTestLoaders[0].collector.settings['address'],
-                         'httpo://thirteenchars123.onion')
+        self.assertEqual(
+            
deck.tasks.ooni['net_test_loaders'][0].collector.settings['address'],
+            'httpo://thirteenchars123.onion'
+        )
 
         self.assertEqual(deck.netTestLoaders[0].localOptions['backend'],
                          '127.0.0.1')
@@ -202,15 +155,15 @@ class TestDeck(BaseTestCase, ConfigTestCase):
         self.deck_file = os.path.join(self.cwd, deck_hash)
         with open(self.deck_file, 'w+') as f:
             f.write(self.dummy_deck_content_with_many_tests)
-        deck = Deck(decks_directory=".")
-        deck.loadDeck(self.deck_file)
+        deck = NGDeck()
+        deck.open(self.deck_file)
 
         self.assertEqual(
-            deck.netTestLoaders[0].localOptions['backend'],
+            deck.tasks[0].ooni['net_test_loader'].localOptions['backend'],
             '1.1.1.1'
         )
         self.assertEqual(
-            deck.netTestLoaders[1].localOptions['backend'],
+            deck.tasks[1].ooni['net_test_loader'].localOptions['backend'],
             '2.2.2.2'
         )
 
@@ -222,58 +175,65 @@ class TestDeck(BaseTestCase, ConfigTestCase):
                           nettest_to_path,
                           "invalid_test")
 
+    @patch('ooni.deck.BouncerClient', MockBouncerClient)
+    @patch('ooni.deck.CollectorClient', MockCollectorClient)
     @defer.inlineCallbacks
     def test_lookup_test_helpers_and_collector_cloudfront(self):
         self.config.advanced.preferred_backend = "cloudfront"
-        deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS,
-                    decks_directory=".")
-        deck.bouncer = MockBouncerClient(FAKE_BOUNCER_ADDRESS)
-        deck._BouncerClient = MockBouncerClient
-        deck._CollectorClient = MockCollectorClient
-        deck.loadDeck(self.deck_file)
-
-        self.assertEqual(len(deck.netTestLoaders[0].missingTestHelpers), 1)
-
-        yield lookup_collector_and_test_helpers(deck.preferred_backend,
-                                                deck.netTestLoaders)
+        deck = NGDeck()
+        deck.open(self.deck_file)
+        first_net_test_loader = deck.tasks[0].ooni['net_test_loader']
+        net_test_loaders = map(lambda task: task.ooni['net_test_loader'],
+                               deck.tasks)
+        self.assertEqual(len(first_net_test_loader.missingTestHelpers), 1)
+
+        yield lookup_collector_and_test_helpers(
+            net_test_loaders=net_test_loaders ,
+            preferred_backend='cloudfront',
+            bouncer=deck.bouncer
+        )
 
         self.assertEqual(
-            deck.netTestLoaders[0].collector.settings['address'],
+            first_net_test_loader.collector.settings['address'],
             'https://address.cloudfront.net'
         )
         self.assertEqual(
-            deck.netTestLoaders[0].collector.settings['front'],
+            first_net_test_loader.collector.settings['front'],
             'front.cloudfront.net'
         )
 
         self.assertEqual(
-            deck.netTestLoaders[0].localOptions['backend'],
+            first_net_test_loader.localOptions['backend'],
             '127.0.0.1'
         )
 
-
+    @patch('ooni.deck.BouncerClient', MockBouncerClient)
+    @patch('ooni.deck.CollectorClient', MockCollectorClient)
     @defer.inlineCallbacks
     def test_lookup_test_helpers_and_collector_https(self):
         self.config.advanced.preferred_backend = "https"
-        deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS,
-                    decks_directory=".")
-        deck.bouncer = MockBouncerClient(FAKE_BOUNCER_ADDRESS)
-        deck._BouncerClient = MockBouncerClient
-        deck._CollectorClient = MockCollectorClient
-        deck.loadDeck(self.deck_file)
+        deck = NGDeck()
+        deck.open(self.deck_file)
+
+        first_net_test_loader = deck.tasks[0].ooni['net_test_loader']
+        net_test_loaders = map(lambda task: task.ooni['net_test_loader'],
+                               deck.tasks)
 
-        self.assertEqual(len(deck.netTestLoaders[0].missingTestHelpers), 1)
+        self.assertEqual(len(first_net_test_loader .missingTestHelpers), 1)
 
-        yield lookup_collector_and_test_helpers(deck.preferred_backend,
-                                                deck.netTestLoaders)
+        yield lookup_collector_and_test_helpers(
+            net_test_loaders=net_test_loaders,
+            preferred_backend='https',
+            bouncer=deck.bouncer
+        )
 
         self.assertEqual(
-            deck.netTestLoaders[0].collector.settings['address'],
+            first_net_test_loader.collector.settings['address'],
             'https://collector.ooni.io'
         )
 
         self.assertEqual(
-            deck.netTestLoaders[0].localOptions['backend'],
+            first_net_test_loader.localOptions['backend'],
             '127.0.0.1'
         )
 
@@ -301,20 +261,65 @@ DECK_DATA = {
         deepcopy(TASK_DATA)
     ]
 }
+
+LEGACY_DECK = """
+- options:
+    annotations: null
+    bouncer: null
+    collector: null
+    no-collector: 0
+    no-geoip: 0
+    no-yamloo: 0
+    reportfile: null
+    subargs: [--flag, --key, value]
+    test_file: manipulation/http_invalid_request_line
+    verbose: 0
+- options:
+    annotations: null
+    bouncer: null
+    collector: null
+    no-collector: 0
+    no-geoip: 0
+    no-yamloo: 0
+    reportfile: null
+    subargs: []
+    test_file: manipulation/http_header_field_manipulation
+    verbose: 0
+- options:
+    annotations: null
+    bouncer: null
+    collector: null
+    no-collector: 0
+    no-geoip: 0
+    no-yamloo: 0
+    reportfile: null
+    subargs: [-f, /path/to/citizenlab-urls-global.txt]
+    test_file: blocking/web_connectivity
+    verbose: 0
+"""
+
 class TestNGDeck(ConfigTestCase):
-    skip = True
     def test_deck_task(self):
-        if self.skip:
-            self.skipTest("Skip is set to true")
-        yield input_store.update("ZZ")
-        deck_task = DeckTask(TASK_DATA)
-        self.assertIsInstance(deck_task.ooni["net_test_loader"],
-                              NetTestLoader)
+        #yield input_store.update("ZZ")
+        #deck_task = DeckTask(TASK_DATA)
+        #self.assertIsInstance(deck_task.ooni["net_test_loader"],
+        #                      NetTestLoader)
+        pass
 
-    @defer.inlineCallbacks
     def test_deck_load(self):
-        if self.skip:
-            self.skipTest("Skip is set to true")
-        yield input_store.update("ZZ")
-        deck = NGDeck(deck_data=DECK_DATA)
-        self.assertEqual(len(deck.tasks), 1)
+        #yield input_store.update("ZZ")
+        #deck = NGDeck(deck_data=DECK_DATA)
+        #self.assertEqual(len(deck.tasks), 1)
+        pass
+
+    def test_convert_legacy_deck(self):
+        legacy_deck = yaml.safe_load(StringIO(LEGACY_DECK))
+        ng_deck = convert_legacy_deck(legacy_deck)
+        self.assertEqual(len(ng_deck['tasks']), 3)
+        task_names = map(lambda task: task['ooni']['test_name'],
+                         ng_deck['tasks'])
+        self.assertItemsEqual(task_names, [
+            "manipulation/http_invalid_request_line",
+            "manipulation/http_header_field_manipulation",
+            "blocking/web_connectivity"
+        ])
diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py
index fe24bf6..6550b3d 100644
--- a/ooni/ui/cli.py
+++ b/ooni/ui/cli.py
@@ -26,19 +26,18 @@ class Options(usage.Options):
 
     optFlags = [["help", "h"],
                 ["no-collector", "n", "Disable writing to collector"],
-                ["no-yamloo", "N", "Disable writing to YAML file"],
+                ["no-njson", "N", "Disable writing to disk"],
                 ["no-geoip", "g", "Disable geoip lookup on start"],
                 ["list", "s", "List the currently installed ooniprobe "
                               "nettests"],
-                ["printdeck", "p", "Print the equivalent deck for the "
-                                   "provided command"],
                 ["verbose", "v", "Show more verbose information"]
                 ]
 
     optParameters = [
-        ["reportfile", "o", None, "Specify the report file name to write to."],
+        ["reportfile", "o", None, "Specify the report file name to write "
+                                  "to."],
         ["testdeck", "i", None, "Specify as input a test deck: a yaml file "
-                                "containing the tests to run and their "
+                                 "containing the tests to run and their "
                                 "arguments."],
         ["collector", "c", None, "Specify the address of the collector for "
                                  "test results. In most cases a user will "
@@ -132,7 +131,8 @@ def director_startup_handled_failures(failure):
                  errors.CouldNotFindTestCollector,
                  errors.ProbeIPUnknown,
                  errors.InvalidInputFile,
-                 errors.ConfigFileIncoherent)
+                 errors.ConfigFileIncoherent,
+                 SystemExit)
 
     if isinstance(failure.value, errors.TorNotRunning):
         log.err("Tor does not appear to be running")
@@ -236,64 +236,71 @@ def setupCollector(global_options, collector_client):
     return collector_client
 
 def createDeck(global_options, url=None):
-    from ooni.nettest import NetTestLoader
-    from ooni.deck import Deck, nettest_to_path
-    from ooni.backend_client import CollectorClient
+    from ooni.deck import NGDeck, subargs_to_options
 
     if url:
         log.msg("Creating deck for: %s" % (url))
 
-    if global_options['no-yamloo']:
-        log.msg("Will not write to a yamloo report file")
-
-    deck = Deck(bouncer=global_options['bouncer'],
-                no_collector=global_options['no-collector'])
-
+    test_deck_path = global_options.pop('testdeck', None)
+    test_name = global_options.pop('test_file', None)
+    no_collector = global_options.pop('no-collector', False)
     try:
-        if global_options['testdeck']:
-            deck.loadDeck(global_options['testdeck'], global_options)
+        if test_deck_path is not None:
+            deck = NGDeck(
+                global_options=global_options,
+                no_collector=no_collector
+            )
+            deck.open(test_deck_path)
         else:
+            deck = NGDeck(
+                global_options=global_options,
+                no_collector=no_collector,
+                arbitrary_paths=True
+            )
             log.debug("No test deck detected")
-            test_file = nettest_to_path(global_options['test_file'], True)
             if url is not None:
                 args = ('-u', url)
             else:
                 args = tuple()
             if any(global_options['subargs']):
                 args = global_options['subargs'] + args
-            net_test_loader = NetTestLoader(args,
-                                            test_file=test_file,
-                                            
annotations=global_options['annotations'])
-            if global_options['collector']:
-                net_test_loader.collector = \
-                    CollectorClient(global_options['collector'])
-            deck.insert(net_test_loader)
+
+            test_options = subargs_to_options(args)
+            test_options['test_name'] = test_name
+            deck.load({
+                "tasks": [
+                    {"ooni": test_options}
+                ]
+            })
     except errors.MissingRequiredOption as option_name:
         log.err('Missing required option: "%s"' % option_name)
         incomplete_net_test_loader = option_name.net_test_loader
         print incomplete_net_test_loader.usageOptions().getUsage()
-        sys.exit(2)
+        raise SystemExit(2)
+
     except errors.NetTestNotFound as path:
         log.err('Requested NetTest file not found (%s)' % path)
-        sys.exit(3)
+        raise SystemExit(3)
+
     except errors.OONIUsageError as e:
         log.err(e)
         print e.net_test_loader.usageOptions().getUsage()
-        sys.exit(4)
+        raise SystemExit(4)
+
     except errors.HTTPSCollectorUnsupported:
         log.err("HTTPS collectors require a twisted version of at least 
14.0.2.")
-        sys.exit(6)
+        raise SystemExit(6)
     except errors.InsecureBackend:
         log.err("Attempting to report to an insecure collector.")
         log.err("To enable reporting to insecure collector set the "
                 "advanced->insecure_backend option to true in "
                 "your ooniprobe.conf file.")
-        sys.exit(7)
+        raise SystemExit(7)
     except Exception as e:
         if config.advanced.debug:
             log.exception(e)
         log.err(e)
-        sys.exit(5)
+        raise SystemExit(5)
 
     return deck
 
@@ -301,45 +308,21 @@ def createDeck(global_options, url=None):
 def runTestWithDirector(director, global_options, url=None, start_tor=True):
     deck = createDeck(global_options, url=url)
 
-    start_tor |= deck.requiresTor
-
-    d = director.start(start_tor=start_tor,
-                       check_incoherences=global_options['check_incoherences'])
-
-    def setup_nettest(_):
+    d = director.start()
+    @defer.inlineCallbacks
+    def post_director_start(_):
         try:
-            return deck.setup()
+            deck.setup()
+            yield deck.run(director)
         except errors.UnableToLoadDeckInput as error:
-            return defer.failure.Failure(error)
+            raise defer.failure.Failure(error)
         except errors.NoReachableTestHelpers as error:
-            return defer.failure.Failure(error)
+            raise defer.failure.Failure(error)
         except errors.NoReachableCollectors as error:
-            return defer.failure.Failure(error)
+            raise defer.failure.Failure(error)
+        except SystemExit as error:
+            raise error
 
-    # Wait until director has started up (including bootstrapping Tor)
-    # before adding tests
-    @defer.inlineCallbacks
-    def post_director_start(_):
-        for net_test_loader in deck.netTestLoaders:
-            # Decks can specify different collectors
-            # for each net test, so that each NetTest
-            # may be paired with a test_helper and its collector
-            # However, a user can override this behavior by
-            # specifying a collector from the command-line (-c).
-            # If a collector is not specified in the deck, or the
-            # deck is a singleton, the default collector set in
-            # ooniprobe.conf will be used
-            collector_client = None
-            if not global_options['no-collector']:
-                collector_client = setupCollector(global_options,
-                                                  net_test_loader.collector)
-
-            yield director.start_net_test_loader(net_test_loader,
-                                                 global_options['reportfile'],
-                                                 collector_client,
-                                                 global_options['no-yamloo'])
-
-    d.addCallback(setup_nettest)
     d.addCallback(post_director_start)
     d.addErrback(director_startup_handled_failures)
     d.addErrback(director_startup_other_failures)
@@ -379,14 +362,7 @@ def runWithDirector(global_options):
             print "Note: Third party tests require an external "\
                   "application to run properly."
 
-        sys.exit(0)
-
-    elif global_options['printdeck']:
-        del global_options['printdeck']
-        print "# Copy and paste the lines below into a test deck to run the 
specified test with the specified arguments"
-        print yaml.safe_dump([{'options': global_options}]).strip()
-
-        sys.exit(0)
+        raise SystemExit(0)
 
     if global_options.get('annotations') is not None:
         global_options['annotations'] = setupAnnotations(global_options)
@@ -427,7 +403,7 @@ def runWithDaemonDirector(global_options):
     except ImportError:
         print "Pika is required for queue connection."
         print "Install with \"pip install pika\"."
-        sys.exit(7)
+        raise SystemExit(7)
 
     director = Director()
 
diff --git a/ooni/ui/web/client/index.html b/ooni/ui/web/client/index.html
index e306ef2..e90ef83 100644
--- a/ooni/ui/web/client/index.html
+++ b/ooni/ui/web/client/index.html
@@ -13,5 +13,5 @@
     <app>
       Loading...
     </app>
-  <script type="text/javascript" 
src="app.bundle.js?7ed7d7510803fa1a4ad8"></script></body>
+  <script type="text/javascript" 
src="app.bundle.js?6b15c1dd202a0f5a80e7"></script></body>
 </html>
diff --git a/scripts/__init__.py b/scripts/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/scripts/set_caps/Makefile b/scripts/set_caps/Makefile
new file mode 100644
index 0000000..bc80843
--- /dev/null
+++ b/scripts/set_caps/Makefile
@@ -0,0 +1,81 @@
+# Wrappers for running ooniprobe as a non-root user.
+#
+# Build-Depends: cython, pythonX.Y-dev, libcap2-bin
+# Depends: libpythonX.Y
+#
+# $ make && make check
+# $ sudo make install # after installing the rest of ooni-probe
+# $ make installcheck_unsafe # runs complete tests as non-root
+#
+# `make` builds a program that has file capabilities set on it. This is just
+# ./ooniprobe compiled into a C program using Cython, so that one can set
+# capabilities directly on the resulting binary. This way, we avoid the need
+# for a separate child python interpreter with its own capabilities. Another
+# advantage is that libpython.so (needed by the program) would be automatically
+# upgraded by the system package manager. The version of python is hard-coded
+# into the wrapper at build time; making this dynamic is possible, but much
+# more complex and not yet implemented.
+#
+# Execution may additionally be limited to a particular unix group by using
+# chgrp(1) and chmod(1) to 'o-x,g+x' after installation.
+#
+
+# GNU Makefile conventions, see 
https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
+prefix = /usr/local
+exec_prefix = $(prefix)
+bindir = $(exec_prefix)/bin
+
+INSTALL = install
+PYTHON = python
+PYTHON_CONFIG = python-config
+CYTHON = cython
+SETCAP = setcap
+
+INSTALL_PROGRAM = $(INSTALL)
+PY_CFLAGS = $(shell $(PYTHON_CONFIG) --cflags)
+PY_LDFLAGS = $(shell $(PYTHON_CONFIG) --ldflags)
+
+BUILDDIR := ./build
+SCRIPTDIR := .
+TESTDIR := ./test
+CAP_SCRIPT := ooniprobe
+CAP_NEEDED := cap_net_admin,cap_net_raw
+
+# Unfortunately cython --embed ignores the arguments in the shebang line
+# So we need to patch the generated code ourselves.
+CYTHON_PRE_MAIN = extern int Py_IgnoreEnvironmentFlag; \
+                  Py_IgnoreEnvironmentFlag++; \
+                  extern int Py_NoUserSiteDirectory; \
+                  Py_NoUserSiteDirectory++;
+
+all: $(BUILDDIR)/$(CAP_SCRIPT)
+
+$(BUILDDIR)/$(CAP_SCRIPT): $(BUILDDIR)/$(CAP_SCRIPT).c Makefile
+       $(CC) $(PY_CFLAGS) $(PY_LDFLAGS) "$<" -o "$@"
+
+$(BUILDDIR)/$(CAP_SCRIPT).c: $(SCRIPTDIR)/$(CAP_SCRIPT) Makefile
+       mkdir -p "$(BUILDDIR)"
+       $(CYTHON) "$<" --embed=CYTHON_MAIN_SENTINEL -Werror -Wextra -o "$@"
+       sed -i \
+         -e 's/\(.*CYTHON_MAIN_SENTINEL.*{\)/\1 $(CYTHON_PRE_MAIN)/g' \
+         -e '/CYTHON_MAIN_SENTINEL[^{]*$$/,/{/s/{/{ $(CYTHON_PRE_MAIN)/g' \
+         -e 's/CYTHON_MAIN_SENTINEL/main/g' "$@"
+
+check: $(BUILDDIR)/$(CAP_SCRIPT)
+       # test that setcapped binary ignores PYTHONPATH
+       BIN="$$(realpath "$<")" && cd "$(TESTDIR)" && PYTHONPATH=. $$BIN 
--version
+
+install: $(BUILDDIR)/$(CAP_SCRIPT)
+       mkdir -p "$(DESTDIR)$(bindir)"
+       $(INSTALL_PROGRAM) -t "$(DESTDIR)$(bindir)" "$(BUILDDIR)/$(CAP_SCRIPT)"
+       $(SETCAP) "$(CAP_NEEDED)"+eip "$(DESTDIR)$(bindir)/$(CAP_SCRIPT)"
+
+installcheck_unsafe: $(BUILDDIR)/$(CAP_SCRIPT)
+       # run a standard check. note that because of hardcoded paths (for 
security)
+       # this can only work after you've installed your development copy
+       "./$<" -i /usr/share/ooni/decks/complete.deck
+
+clean:
+       rm -rf "$(BUILDDIR)"
+
+.PHONY: clean all check install installcheck%
diff --git a/scripts/set_caps/__init__.py b/scripts/set_caps/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/scripts/set_caps/test/__init__.py 
b/scripts/set_caps/test/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/scripts/set_caps/test/ooni/__init__.py 
b/scripts/set_caps/test/ooni/__init__.py
new file mode 100644
index 0000000..f2cd353
--- /dev/null
+++ b/scripts/set_caps/test/ooni/__init__.py
@@ -0,0 +1 @@
+raise ValueError("test failed! wrapper did not ignore polluted PWD. either the 
wrapper is faulty, or ooni is still unpatched (Tor bug #13581)")



_______________________________________________
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

Reply via email to