[tor-commits] [ooni-probe/master] Add some basic unittests for the ScheduledTasks

[art]

commit 5723658b20d834dfd02b04264bf3f9e4cd8c9f50
Author: Arturo Filastò 
Date:   Tue Sep 6 16:50:03 2016 +0200

Add some basic unittests for the ScheduledTasks
---
 ooni/agent/scheduler.py  |  6 +++---
 ooni/settings.py |  2 --
 ooni/tests/test_scheduler.py | 51 
 3 files changed, 54 insertions(+), 5 deletions(-)

diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 21dd1cd..98b395b 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -53,7 +53,8 @@ class ScheduledTask(object):
 schedule = None
 identifier = None
 
-def __init__(self, schedule=None, identifier=None):
+def __init__(self, schedule=None, identifier=None,
+ scheduler_directory=config.scheduler_directory):
 if schedule is not None:
 self.schedule = schedule
 if identifier is not None:
@@ -61,7 +62,6 @@ class ScheduledTask(object):
 
 assert self.identifier is not None, "self.identifier must be set"
 assert self.schedule is not None, "self.schedule must be set"
-scheduler_directory = config.scheduler_directory
 
 self._last_run = FilePath(scheduler_directory).child(self.identifier)
 self._last_run_lock = FileSystemlockAndMutex(
@@ -91,7 +91,7 @@ class ScheduledTask(object):
 out_file.write(current_time.strftime(self._time_format))
 
 def task(self):
-raise NotImplemented
+raise NotImplementedError
 
 def first_run(self):
 """
diff --git a/ooni/settings.py b/ooni/settings.py
index a1c55fc..409e81e 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -3,7 +3,6 @@ import sys
 import yaml
 import errno
 import getpass
-from pkg_resources import parse_version
 from ConfigParser import SafeConfigParser
 
 from twisted.internet import defer, reactor
@@ -11,7 +10,6 @@ from twisted.internet.endpoints import TCP4ClientEndpoint
 
 from os.path import abspath, expanduser
 
-from ooni import __version__ as ooniprobe_version
 from ooni.utils import Storage, log, get_ooni_root
 
 CONFIG_FILE_TEMPLATE = """\
diff --git a/ooni/tests/test_scheduler.py b/ooni/tests/test_scheduler.py
new file mode 100644
index 000..cb90a30
--- /dev/null
+++ b/ooni/tests/test_scheduler.py
@@ -0,0 +1,51 @@
+import os
+import shutil
+import tempfile
+
+from twisted.internet import defer
+from twisted.trial import unittest
+
+from ooni.agent.scheduler import ScheduledTask, DidNotRun
+
+class TestScheduler(unittest.TestCase):
+def test_scheduled_task(self):
+schedule = "@daily"
+identifier = "dummy"
+scheduler_directory = tempfile.mkdtemp()
+scheduled_task = ScheduledTask(schedule=schedule,
+   identifier=identifier,
+   scheduler_directory=scheduler_directory
+)
+self.assertEqual(scheduled_task.should_run, True)
+self.assertFailure(scheduled_task.run(), NotImplementedError)
+self.assertEqual(scheduled_task.should_run, True)
+shutil.rmtree(scheduler_directory)
+
+@defer.inlineCallbacks
+def test_call_twice_scheduled_task(self):
+"""
+If we call the scheduled task twice in a row the second time it will 
not run.
+Tests for possible race conditions.
+"""
+scheduler_directory = tempfile.mkdtemp()
+spam_path = os.path.join(scheduler_directory, 'spam.txt')
+class DummyST(ScheduledTask):
+def task(self):
+with open(spam_path, 'w') as out_file:
+out_file.write("I ran\n")
+
+schedule = "@daily"
+identifier = "dummy"
+dummy_st = DummyST(schedule=schedule,
+   identifier=identifier,
+   scheduler_directory=scheduler_directory
+)
+
+dummy_st.run()
+yield self.assertFailure(dummy_st.run(), DidNotRun)
+
+with open(spam_path) as in_file:
+self.assertEqual(len(in_file.readlines()), 1)
+
+self.assertEqual(dummy_st.should_run, False)
+shutil.rmtree(scheduler_directory)



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Update the updater

[art]

commit 8fa6e1ef7d7e1d8d7140e6171a7debabeffab908
Author: Arturo Filastò 
Date:   Wed Sep 7 13:21:27 2016 +0200

Update the updater
---
 data/updater.py | 14 --
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/data/updater.py b/data/updater.py
index 0f423ab..10086f5 100755
--- a/data/updater.py
+++ b/data/updater.py
@@ -20,8 +20,8 @@ from subprocess import check_output, check_call, 
CalledProcessError
 UPDATE_BASE_URL = 
"https://github.com/OpenObservatory/lepidopter-update/releases/download/;
 
 CURRENT_VERSION_PATH = "/etc/lepidopter-update/version"
-UPDATER_PATH = "/opt/ooni/updater/versions/"
-SCRIPT_INSTALL_PATH = "/opt/ooni/updater/updater.py"
+UPDATER_PATH = "/opt/ooni/lepidopter-update/versions/"
+SCRIPT_INSTALL_PATH = "/opt/ooni/lepidopter-update/updater.py"
 
 SYSTEMD_SCRIPT_PATH = "/etc/systemd/system/lepidopter-update.service"
 SYSTEMD_SCRIPT = """\
@@ -30,14 +30,16 @@ Description=lepidopter-update service
 
 [Service]
 Type=simple
-ExecStart={0} --log-file /var/log/lepidopter-update.log update --watch
+Environment="UPDATER={0}"
+ExecStart=$UPDATER --log-file /var/log/ooni/lepidopter-update.log update 
--watch
 TimeoutStartSec=300
 Restart=on-failure
+
 [Install]
 WantedBy=multi-user.target
 """.format(SCRIPT_INSTALL_PATH)
 
-PUBLIC_KEY_PATH = "/opt/ooni/updater/public.asc"
+PUBLIC_KEY_PATH = "/opt/ooni/lepidopter-update/public.asc"
 PUBLIC_KEY = """\
 -BEGIN PGP PUBLIC KEY BLOCK-
 Comment: GPGTools - https://gpgtools.org
@@ -306,8 +308,8 @@ def install(args):
 with open(SYSTEMD_SCRIPT_PATH, "w") as out_file:
 out_file.write(SYSTEMD_SCRIPT)
 
-check_call(["systemctl", "enable", "lepidopter-updater"])
-check_call(["systemctl", "start", "lepidopter-updater"])
+check_call(["systemctl", "enable", "lepidopter-update"])
+check_call(["systemctl", "start", "lepidopter-update"])
 
 class InvalidLogLevel(Exception):
 pass



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Better permissions handling for logfile and folder

[art]

commit 73b17d70ad8ac535ddebe36ca1b1d4c6277781d1
Author: Arturo Filastò 
Date:   Fri Sep 2 15:17:24 2016 +0200

Better permissions handling for logfile and folder
---
 ooni/utils/log.py | 15 +++
 1 file changed, 15 insertions(+)

diff --git a/ooni/utils/log.py b/ooni/utils/log.py
index 1aba8dd..67079e0 100644
--- a/ooni/utils/log.py
+++ b/ooni/utils/log.py
@@ -1,5 +1,6 @@
 import os
 import sys
+import errno
 import codecs
 import logging
 
@@ -132,6 +133,20 @@ class OONILogger(object):
 logfile = os.path.expanduser(config.basic.logfile)
 
 log_folder = os.path.dirname(logfile)
+if not os.access(log_folder, os.W_OK):
+# If we don't have permissions to write to the log_folder,
+# write to running dir.
+log_folder = config.running_path
+logfile = os.path.join(log_folder, "ooniprobe.log")
+
+try:
+os.makedirs(log_folder)
+except OSError as ose:
+if ose.errno == errno.EEXIST and os.path.isdir(log_folder):
+pass
+else:
+raise
+
 log_filename = os.path.basename(logfile)
 file_log_level = levels.get(config.basic.loglevel,
 levels['INFO'])



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Configure default decks when the scheduler is run for the first time.

[art]

commit 4e777252050d8b0266ce726ee8e43abfb6eaeb21
Author: Arturo Filastò 
Date:   Fri Sep 2 17:36:14 2016 +0200

Configure default decks when the scheduler is run for the first time.

* Add hook for running an operation when the task is run for the first time.
---
 ooni/agent/scheduler.py | 28 ++--
 ooni/deck/store.py  |  3 +++
 ooni/director.py|  2 +-
 ooni/settings.py| 33 -
 4 files changed, 30 insertions(+), 36 deletions(-)

diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 54551f6..7c2212b 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -12,7 +12,7 @@ from ooni.scripts import oonireport
 from ooni import resources
 from ooni.utils import log, SHORT_DATE
 from ooni.utils.files import human_size_to_bytes, directory_usage
-from ooni.deck.store import input_store, deck_store
+from ooni.deck.store import input_store, deck_store, DEFAULT_DECKS
 from ooni.settings import config
 from ooni.contrib import croniter
 from ooni.geoip import probe_ip
@@ -41,6 +41,10 @@ class FileSystemlockAndMutex(object):
 self._fs_lock.unlock()
 self._mutex.release()
 
+# We use this date to indicate that the scheduled task has never run.
+# Easter egg, try to see what is special about this date :)?
+CANARY_DATE = datetime(1957, 8, 4)
+
 class DidNotRun(Exception):
 pass
 
@@ -76,7 +80,7 @@ class ScheduledTask(object):
 def last_run(self):
 self._last_run.restat(False)
 if not self._last_run.exists():
-return datetime.fromtimestamp(0)
+return CANARY_DATE
 with self._last_run.open('r') as in_file:
 date_str = in_file.read()
 return datetime.strptime(date_str, self._time_format)
@@ -89,6 +93,13 @@ class ScheduledTask(object):
 def task(self):
 raise NotImplemented
 
+def first_run(self):
+"""
+This hook is called if it's the first time a particular scheduled
+operation is run.
+"""
+pass
+
 @defer.inlineCallbacks
 def run(self):
 yield self._last_run_lock.acquire()
@@ -96,6 +107,8 @@ class ScheduledTask(object):
 self._last_run_lock.release()
 raise DidNotRun
 try:
+if self.last_run == CANARY_DATE:
+yield defer.maybeDeferred(self.first_run)
 yield self.task()
 self._update_last_run()
 except:
@@ -240,6 +253,13 @@ class RefreshDeckList(ScheduledTask):
 self.scheduler = scheduler
 super(RefreshDeckList, self).__init__(schedule, identifier)
 
+def first_run(self):
+"""
+On first run we enable the default decks.
+"""
+for deck_id in DEFAULT_DECKS:
+deck_store.enable(deck_id)
+
 def task(self):
 self.scheduler.refresh_deck_list()
 
@@ -302,6 +322,10 @@ class SchedulerService(service.MultiService):
 if isinstance(scheduled_task, RunDeck):
 self._scheduled_tasks.remove(scheduled_task)
 
+if not config.is_initialized():
+# Disable scheduling measurements if we are not initialized.
+return
+
 for deck_id, deck in deck_store.list_enabled():
 if deck.schedule is None:
 continue
diff --git a/ooni/deck/store.py b/ooni/deck/store.py
index c938d56..bd852c7 100644
--- a/ooni/deck/store.py
+++ b/ooni/deck/store.py
@@ -11,6 +11,9 @@ from ooni.otime import timestampNowISO8601UTC
 from ooni.resources import check_for_update
 from ooni.settings import config
 
+# These are the decks to be run by default.
+DEFAULT_DECKS = ['web-full']
+
 class InputNotFound(Exception):
 pass
 
diff --git a/ooni/director.py b/ooni/director.py
index ab31bf5..cb2bdf0 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -398,7 +398,7 @@ class Director(object):
 self._tor_starting.callback(self._tor_state)
 except Exception as exc:
 log.err("Failed to start tor")
-log.exc(exc)
+log.exception(exc)
 self._tor_starting.errback(Failure(exc))
 
 elif config.tor.control_port and config.tor_state is None:
diff --git a/ooni/settings.py b/ooni/settings.py
index 9f24f9b..a1c55fc 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -266,31 +266,6 @@ class OConfig(object):
 with open(initialized_path, 'w+'): pass
 
 @property
-def last_run_version(self):
-"""
-:return: Version identifying the last run version of ooniprobe.
-"""
-last_run_version_path = os.path.join(
-self.running_path, "last_run_version"
-)
-if not os.path.exists(last_run_version_path):
-return parse_version("0")
-with open(last_run_version_path) as in_file:
-last_run_version = in_file.read()
-return parse_version(last_run_version)

[tor-commits] [ooni-probe/master] When we talk about time ALWAYS use UTC

[art]

commit 46027a39a768efa6beabac9af62765970762f6bf
Author: Arturo Filastò 
Date:   Tue Sep 13 13:21:36 2016 +0200

When we talk about time ALWAYS use UTC
---
 ooni/agent/scheduler.py | 6 --
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 0a6cc0f..d6fbc31 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -50,11 +50,13 @@ class FileSystemlockAndMutex(object):
 
 # We use this date to indicate that the scheduled task has never run.
 # Easter egg, try to see what is special about this date :)?
-CANARY_DATE = datetime(1957, 8, 4)
+CANARY_DATE = datetime(1957, 8, 4, tzinfo=tz.tzutc())
+
 
 class DidNotRun(Exception):
 pass
 
+
 class ScheduledTask(object):
 _time_format = "%Y-%m-%dT%H:%M:%SZ"
 schedule = None
@@ -80,7 +82,7 @@ class ScheduledTask(object):
 
 @property
 def should_run(self):
-current_time = datetime.utcnow()
+current_time = datetime.utcnow().replace(tzinfo=tz.tzutc())
 next_cycle = croniter(self.schedule, self.last_run).get_next(datetime)
 if next_cycle <= current_time:
 return True



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Update updater script

[art]

commit 1d5d892181bf06f21b7c8df8566eb0906212382b
Author: Arturo Filastò 
Date:   Tue Sep 13 12:32:55 2016 +0200

Update updater script
---
 data/updater.py | 7 ++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/data/updater.py b/data/updater.py
index 65983aa..f7225dd 100755
--- a/data/updater.py
+++ b/data/updater.py
@@ -198,9 +198,12 @@ def perform_update(version, skip_verification=False):
 
 try:
 logging.info("Running install script")
+if updater.__version__ != str(version):
+logging.error("There is a version mismatch in the updater file. 
This could be a sign of a replay attack.")
+raise UpdateFailed
 updater.run()
 except Exception:
-logging.error("Failed to run the version update script for version 
{0}".format(version))
+logging.exception("Failed to run the version update script for version 
{0}".format(version))
 raise UpdateFailed
 
 current_version_dir = os.path.dirname(CURRENT_VERSION_PATH)
@@ -214,6 +217,8 @@ def perform_update(version, skip_verification=False):
 with open(CURRENT_VERSION_PATH, "w+") as out_file:
 out_file.write(str(version))
 
+logging.info("Updated to version {0}".format(version))
+
 def update_to_version(from_version, to_version, skip_verification=False):
 versions = range(from_version + 1, to_version + 1)
 for version in versions:



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Improve sanity checks on tor_data_dir

[art]

commit a731364e988798ebeb12e52eff1b1c11e18cdb2c
Author: Arturo Filastò 
Date:   Wed Sep 7 16:49:19 2016 +0200

Improve sanity checks on tor_data_dir
---
 ooni/utils/onion.py | 40 
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py
index 6e0d906..10f80a3 100644
--- a/ooni/utils/onion.py
+++ b/ooni/utils/onion.py
@@ -1,6 +1,8 @@
 import os
 import re
 import pwd
+import fcntl
+import errno
 import string
 import StringIO
 import subprocess
@@ -214,6 +216,25 @@ def get_client_transport(transport):
 
 raise UninstalledTransport
 
+def is_tor_data_dir_usable(tor_data_dir):
+"""
+Checks if the Tor data dir specified is usable. This means that
+ it is not being locked and we have permissions to write to it.
+"""
+if not os.path.exists(tor_data_dir):
+return True
+
+try:
+fcntl.flock(open(os.path.join(tor_data_dir, 'lock'), 'w'),
+fcntl.LOCK_EX | fcntl.LOCK_NB)
+return True
+except (IOError, OSError) as err:
+if err.errno == errno.EACCES:
+# Permission error
+return False
+elif err.errno == errno.EAGAIN:
+# File locked
+return False
 
 def get_tor_config():
 tor_config = TorConfig()
@@ -227,11 +248,22 @@ def get_tor_config():
 
 if config.tor.data_dir:
 data_dir = os.path.expanduser(config.tor.data_dir)
-
-if not os.path.exists(data_dir):
-log.debug("%s does not exist. Creating it." % data_dir)
+# We only use the Tor data dir specified in the config file if
+# 1. It is not locked (i.e. another process is using it)
+# 2. We have write permissions to it
+data_dir_usable = is_tor_data_dir_usable(data_dir)
+try:
 os.makedirs(data_dir)
-tor_config.DataDirectory = data_dir
+log.debug("%s does not exist. Creating it." % data_dir)
+except OSError as ose:
+if ose.errno == errno.EEXIST:
+pass
+elif ose.errno == errno.EACCESS:
+data_dir_usable = False
+else:
+raise
+if data_dir_usable:
+tor_config.DataDirectory = data_dir
 
 if config.tor.bridges:
 tor_config.UseBridges = 1



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Don't override the report_id

[art]

commit 406c60567a16f6a795602f59c7ec5a1f3e5bcce2
Author: Arturo Filastò 
Date:   Tue Sep 13 17:16:10 2016 +0200

Don't override the report_id
---
 ooni/reporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ooni/reporter.py b/ooni/reporter.py
index cfdce08..b149d49 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -608,7 +608,7 @@ class Report(object):
 if self.collector_client:
 self.oonib_reporter = OONIBReporter(self.test_details,
 self.collector_client)
-self.test_details['report_id'] = yield self.open_oonib_reporter()
+yield self.open_oonib_reporter()
 
 if not self.no_njson:
 self.njson_reporter = NJSONReporter(self.test_details,



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Bugfix in updater

[art]

commit 96c7b6103023f4968141eb707266bf2bfa167368
Author: Arturo Filastò 
Date:   Wed Sep 7 13:36:43 2016 +0200

Bugfix in updater
---
 data/updater.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/data/updater.py b/data/updater.py
index 10086f5..65983aa 100755
--- a/data/updater.py
+++ b/data/updater.py
@@ -30,8 +30,7 @@ Description=lepidopter-update service
 
 [Service]
 Type=simple
-Environment="UPDATER={0}"
-ExecStart=$UPDATER --log-file /var/log/ooni/lepidopter-update.log update 
--watch
+ExecStart={0} --log-file /var/log/ooni/lepidopter-update.log update --watch
 TimeoutStartSec=300
 Restart=on-failure
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Check permissions on logfile as well

[art]

commit bfc0a51a0b63ceb1d07812f2a7edc6a63847ea90
Author: Arturo Filastò 
Date:   Tue Sep 6 17:09:18 2016 +0200

Check permissions on logfile as well

* Handle the case where the measurement directory doesn't exist yet
---
 ooni/measurements.py | 2 ++
 ooni/utils/log.py| 7 ---
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/ooni/measurements.py b/ooni/measurements.py
index fd722ee..294b182 100644
--- a/ooni/measurements.py
+++ b/ooni/measurements.py
@@ -128,6 +128,8 @@ def get_summary(measurement_id):
 def list_measurements(compute_size=False):
 measurements = []
 measurement_path = FilePath(config.measurements_directory)
+if not measurement_path.exists():
+return measurements
 for measurement_id in measurement_path.listdir():
 try:
 measurements.append(get_measurement(measurement_id, compute_size))
diff --git a/ooni/utils/log.py b/ooni/utils/log.py
index 67079e0..93e6b31 100644
--- a/ooni/utils/log.py
+++ b/ooni/utils/log.py
@@ -133,9 +133,10 @@ class OONILogger(object):
 logfile = os.path.expanduser(config.basic.logfile)
 
 log_folder = os.path.dirname(logfile)
-if not os.access(log_folder, os.W_OK):
-# If we don't have permissions to write to the log_folder,
-# write to running dir.
+if (not os.access(log_folder, os.W_OK) or
+(os.path.exists(logfile) and not os.access(logfile, os.W_OK))):
+# If we don't have permissions to write to the log_folder or
+# logfile.
 log_folder = config.running_path
 logfile = os.path.join(log_folder, "ooniprobe.log")
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] The updater needs to be updated

[art]

commit 5f3e8e311013d76db1dc62b21f66f0f3ce371c44
Author: Arturo Filastò 
Date:   Tue Sep 13 13:00:11 2016 +0200

The updater needs to be updated
---
 data/updater.py   | 6 +++---
 ooni/ui/web/client/index.html | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/data/updater.py b/data/updater.py
index f7225dd..4cacf3d 100755
--- a/data/updater.py
+++ b/data/updater.py
@@ -133,7 +133,7 @@ class InvalidPublicKey(Exception):
 pass
 
 
-def verify_file(signature_path, signer_pk_path):
+def verify_file(signature_path, file_path, signer_pk_path):
 tmp_dir = tempfile.mkdtemp()
 tmp_key = os.path.join(tmp_dir, "signing-key.gpg")
 
@@ -148,7 +148,7 @@ def verify_file(signature_path, signer_pk_path):
 output = check_output(["gpg", "--batch", "--status-fd", "1",
"--no-default-keyring", "--keyring",
tmp_key, "--trust-model", "always",
-   "--verify", signature_path])
+   "--verify", signature_path, file_path])
 except CalledProcessError:
 raise InvalidSignature
 
@@ -188,7 +188,7 @@ def perform_update(version, skip_verification=False):
 
 if skip_verification is not True:
 try:
-verify_file(updater_sig_path, PUBLIC_KEY_PATH)
+verify_file(updater_sig_path, updater_path, PUBLIC_KEY_PATH)
 except InvalidSignature:
 logging.error("Found an invalid signature. Bailing")
 raise UpdateFailed
diff --git a/ooni/ui/web/client/index.html b/ooni/ui/web/client/index.html
index ecb4cdb..357b6eb 100644
--- a/ooni/ui/web/client/index.html
+++ b/ooni/ui/web/client/index.html
@@ -13,5 +13,5 @@
 
   Loading...
 
-  
+  
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Bugfixes to quota management

[art]

commit e6b6f023681bd4e04efa7eb896f2daf9291451ef
Author: Arturo Filastò 
Date:   Mon Sep 5 18:32:45 2016 +0200

Bugfixes to quota management
---
 ooni/agent/scheduler.py | 10 +-
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 3cbd29d..21dd1cd 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -176,13 +176,13 @@ class CheckMeasurementQuota(ScheduledTask):
 if config.basic.measurement_quota is None:
 return
 maximum_bytes = human_size_to_bytes(config.basic.measurement_quota)
-available_bytes = directory_usage(config.measurements_directory)
+used_bytes = directory_usage(config.measurements_directory)
 warning_path = os.path.join(config.running_path, 'quota_warning')
 
-if (float(available_bytes) / float(maximum_bytes)) >= self._warn_when:
+if (float(used_bytes) / float(maximum_bytes)) >= self._warn_when:
 log.warn("You are about to reach the maximum allowed quota. Be 
careful")
 with open(warning_path, "w") as out_file:
-out_file.write("{0} {1}".format(available_bytes,
+out_file.write("{0} {1}".format(used_bytes,
 maximum_bytes))
 else:
 try:
@@ -191,12 +191,12 @@ class CheckMeasurementQuota(ScheduledTask):
 if ose.errno != errno.ENOENT:
 raise
 
-if float(available_bytes) < float(maximum_bytes):
+if float(used_bytes) < float(maximum_bytes):
 # We are within the allow quota exit.
 return
 
 # We should begin to delete old reports
-amount_to_delete = float(maximum_bytes) - float(available_bytes)
+amount_to_delete = float(used_bytes) - float(maximum_bytes)
 amount_deleted = 0
 measurement_path = FilePath(config.measurements_directory)
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Implement retry of director startup with bounded exponential binary backoff

[art]

commit 4f80a3bb07df8d742cb79f8ca918734412159422
Author: Arturo Filastò 
Date:   Thu Sep 8 16:17:28 2016 +0200

Implement retry of director startup with bounded exponential binary backoff
---
 ooni/director.py  |  7 +++
 ooni/geoip.py |  8 +++-
 ooni/ui/web/server.py | 29 -
 3 files changed, 38 insertions(+), 6 deletions(-)

diff --git a/ooni/director.py b/ooni/director.py
index cb2bdf0..1807e65 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -203,11 +203,9 @@ class Director(object):
 yield probe_ip.lookup()
 self.notify(DirectorEvent("success", "Looked up probe IP"))
 
-self.notify(DirectorEvent("success",
-  "Running system tasks"))
+self.notify(DirectorEvent("success", "Running system tasks"))
 yield run_system_tasks(no_input_store=not create_input_store)
-self.notify(DirectorEvent("success",
-  "Ran system tasks"))
+self.notify(DirectorEvent("success", "Ran system tasks"))
 
 @defer.inlineCallbacks
 def start(self, start_tor=False, check_incoherences=True,
@@ -218,6 +216,7 @@ class Director(object):
 self._director_starting.callback(self._director_state)
 except Exception as exc:
 self._director_starting.errback(Failure(exc))
+raise
 
 @property
 def measurementSuccessRatio(self):
diff --git a/ooni/geoip.py b/ooni/geoip.py
index f271790..40fad25 100644
--- a/ooni/geoip.py
+++ b/ooni/geoip.py
@@ -181,12 +181,17 @@ class ProbeIP(object):
 self._state = INITIAL
 self._looking_up = defer.Deferred()
 self._looking_up.addCallback(self._looked_up)
+self._looking_up.addErrback(self._lookup_failed)
 
 def _looked_up(self, result):
 self._last_lookup = time.time()
 self._reset_state()
 return result
 
+def _lookup_failed(self, failure):
+self._reset_state()
+return failure
+
 def resolveGeodata(self):
 from ooni.settings import config
 
@@ -227,8 +232,9 @@ class ProbeIP(object):
 self.resolveGeodata()
 self._looking_up.callback(self.address)
 defer.returnValue(self.address)
-except Exception:
+except Exception as exc:
 log.msg("Unable to lookup the probe IP via GeoIPService")
+self._looking_up.errback(defer.failure.Failure(exc))
 raise
 
 @defer.inlineCallbacks
diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index 26bfd47..aed9951 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -4,6 +4,7 @@ import os
 import json
 import errno
 import string
+import random
 from functools import wraps
 from random import SystemRandom
 
@@ -150,6 +151,7 @@ class WebUIAPI(object):
 _enable_xsrf_protection = True
 
 def __init__(self, config, director, scheduler, _reactor=reactor):
+self._reactor = reactor
 self.director = director
 self.scheduler = scheduler
 
@@ -165,6 +167,12 @@ class WebUIAPI(object):
 self._director_started = False
 self._is_initialized = config.is_initialized()
 
+# We use exponential backoff to trigger retries of the startup of
+# the director.
+self._director_startup_retries = 0
+# Maximum delay should be 6 hours
+self._director_max_retry_delay = 6*60*60
+
 self.status_poller = LongPoller(
 self._long_polling_timeout, _reactor)
 self.director_event_poller = LongPoller(
@@ -179,9 +187,11 @@ class WebUIAPI(object):
 self.start_director()
 
 def start_director(self):
+log.debug("Starting director")
 d = self.director.start()
 
 d.addCallback(self.director_started)
+d.addErrback(self.director_startup_failed)
 d.addBoth(lambda _: self.status_poller.notify())
 
 @property
@@ -208,7 +218,24 @@ class WebUIAPI(object):
 log.debug("Handling event {0}".format(event.type))
 self.director_event_poller.notify(event)
 
+def director_startup_failed(self, failure):
+self._director_startup_retries += 1
+
+# We delay the startup using binary exponential backoff with an
+# upper bound.
+startup_delay = random.uniform(
+0, min(2**self._director_startup_retries,
+   self._director_max_retry_delay)
+)
+log.err("Failed to start the director, "
+"retrying in {0}s".format(startup_delay))
+self._reactor.callLater(
+startup_delay,
+self.start_director
+)
+
 def director_started(self, _):
+log.debug("Started director")
 self._director_started = True
 
 @app.handle_errors(NotFound)
@@ -435,7 +462,7 @@ class WebUIAPI(object):
 deck.load(deck_data)
 

[tor-commits] [ooni-probe/master] Address feedback from @bassosimone

[art]

commit 47b4587d22993343a26aef6eef3667fe959e005e
Author: Arturo Filastò 
Date:   Wed Sep 14 17:03:21 2016 +0200

Address feedback from @bassosimone
---
 Vagrantfile|  5 -
 data/ooniprobe.conf.sample |  2 +-
 ooni/agent/scheduler.py| 24 +++-
 ooni/deck/backend.py   |  4 ++--
 ooni/deck/deck.py  |  2 +-
 ooni/director.py   |  6 +-
 ooni/settings.py   |  2 +-
 7 files changed, 29 insertions(+), 16 deletions(-)

diff --git a/Vagrantfile b/Vagrantfile
index 8ff0bbd..71f7c7a 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -81,9 +81,4 @@ Vagrant.configure("2") do |config|
 end
   end
 
-  config.vm.define "testing" do |testing|
-testing.vm.network "forwarded_port", guest: 8842, host: 8142
-testing.vm.synced_folder ".", "/data/ooni-probe"
-  end
-
 end
diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample
index f4c0812..ee9a2c9 100644
--- a/data/ooniprobe.conf.sample
+++ b/data/ooniprobe.conf.sample
@@ -3,7 +3,7 @@
 # Keep in mind that indentation matters.
 
 basic:
-# Where OONIProbe should be writing it's log file
+# Where OONIProbe should be writing its log file
 logfile: /var/log/ooniprobe.log
 loglevel: WARNING
 privacy:
diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 2797fba..c86e3b6 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -24,9 +24,12 @@ from ooni.measurements import list_measurements
 class FileSystemlockAndMutex(object):
 """
 This is a lock that is both a mutex lock and also on filesystem.
-When you acquire it, it will first block on the mutex lock and then
-once that is released it will attempt to acquire the lock on the
-filesystem.
+When you acquire it, it will first acquire the mutex lock and then
+acquire the filesystem lock. The release order is inverted.
+
+This is to avoid concurrent usage within the same process. When using it
+concurrently the mutex lock will block before the filesystem lock is
+acquired.
 
 It's a way to support concurrent usage of the DeferredFilesystemLock
 without races.
@@ -291,7 +294,7 @@ class SendHeartBeat(ScheduledTask):
 # XXX implement this
 pass
 
-# Order mattters
+# Order matters
 SYSTEM_TASKS = [
 UpdateInputsAndResources
 ]
@@ -303,7 +306,10 @@ def run_system_tasks(no_input_store=False):
 
 if no_input_store:
 log.debug("Not updating the inputs")
-task_classes.remove(UpdateInputsAndResources)
+try:
+task_classes.remove(UpdateInputsAndResources)
+except ValueError:
+pass
 
 for task_class in task_classes:
 task = task_class()
@@ -337,7 +343,15 @@ class SchedulerService(service.MultiService):
 self._scheduled_tasks.remove(task)
 
 def refresh_deck_list(self):
+"""
+This checks if there are some decks that have been enabled and
+should be scheduled as periodic tasks to run on the next scheduler
+cycle and if some have been disabled and should not be run.
 
+It does so by listing the enabled decks and checking if the enabled
+ones are already scheduled or if some of the scheduled ones are not
+amongst the enabled decks.
+"""
 to_enable = []
 for deck_id, deck in deck_store.list_enabled():
 if deck.schedule is None:
diff --git a/ooni/deck/backend.py b/ooni/deck/backend.py
index b2df9bc..a39492c 100644
--- a/ooni/deck/backend.py
+++ b/ooni/deck/backend.py
@@ -16,6 +16,8 @@ def sort_addresses_by_priority(priority_address,
 'address': priority_address,
 'type': backend_type
 }
+# We prefer an onion collector to an https collector to a cloudfront
+# collector to a plaintext collector
 address_priority = ['onion', 'https', 'cloudfront', 'http']
 address_priority.remove(preferred_backend)
 address_priority.insert(0, preferred_backend)
@@ -73,8 +75,6 @@ def get_reachable_test_helper(test_helper_name, 
test_helper_address,
 @defer.inlineCallbacks
 def get_reachable_collector(collector_address, collector_alternate,
 preferred_backend):
-# We prefer onion collector to https collector to cloudfront
-# collectors to plaintext collectors
 for collector_settings in sort_addresses_by_priority(
 collector_address,
 collector_alternate,
diff --git a/ooni/deck/deck.py b/ooni/deck/deck.py
index 3fc10ab..59946ab 100644
--- a/ooni/deck/deck.py
+++ b/ooni/deck/deck.py
@@ -25,7 +25,7 @@ def resolve_file_path(v, prepath=None):
 if v.startswith("$"):
 # This raises InputNotFound and we let it carry onto the caller
 return input_store.get(v[1:])["filepath"]
-elif prepath is not None and (not os.path.isabs(v)):
+if prepath is not None and (not os.path.isabs(v)):
 return FilePath(prepath).preauthChild(v).path
 return v
 

[tor-commits] [ooni-probe/master] Add more unittests for the scheduler

[art]

commit 4865ade8dc39d924e6a14e00c9b5a9e741f50abc
Author: Arturo Filastò 
Date:   Mon Sep 12 14:24:41 2016 +0200

Add more unittests for the scheduler

* Make the logic for triggering scheduled deck tasks more robust
---
 ooni/agent/scheduler.py  | 49 ++--
 ooni/tests/test_scheduler.py | 28 -
 2 files changed, 65 insertions(+), 12 deletions(-)

diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 98b395b..4002369 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -1,6 +1,7 @@
 import os
 import errno
 
+from hashlib import md5
 from datetime import datetime
 
 from twisted.application import service
@@ -41,6 +42,10 @@ class FileSystemlockAndMutex(object):
 self._fs_lock.unlock()
 self._mutex.release()
 
+@property
+def locked(self):
+return self._mutex.locked or self._fs_lock.locked
+
 # We use this date to indicate that the scheduled task has never run.
 # Easter egg, try to see what is special about this date :)?
 CANARY_DATE = datetime(1957, 8, 4)
@@ -68,6 +73,9 @@ class ScheduledTask(object):
 FilePath(scheduler_directory).child(self.identifier + ".lock").path
 )
 
+def cancel(self):
+self._last_run_lock.release()
+
 @property
 def should_run(self):
 current_time = datetime.utcnow()
@@ -233,7 +241,8 @@ class RunDeck(ScheduledTask):
 def __init__(self, director, deck_id, schedule):
 self.deck_id = deck_id
 self.director = director
-identifier = 'run-deck-' + deck_id
+# We use as identifier also the schedule time
+identifier = 'run-deck-' + deck_id + '-' + md5(schedule).hexdigest()
 super(RunDeck, self).__init__(schedule, identifier)
 
 @defer.inlineCallbacks
@@ -316,21 +325,39 @@ class SchedulerService(service.MultiService):
 def schedule(self, task):
 self._scheduled_tasks.append(task)
 
-def refresh_deck_list(self):
-# Deletes all the RunDeck tasks and reschedules only the ones that
-# are enabled.
-for scheduled_task in self._scheduled_tasks[:]:
-if isinstance(scheduled_task, RunDeck):
-self._scheduled_tasks.remove(scheduled_task)
+def unschedule(self, task):
+# We first cancel the task so the run lock is deleted
+task.cancel()
+self._scheduled_tasks.remove(task)
 
-if not config.is_initialized():
-# Disable scheduling measurements if we are not initialized.
-return
+def refresh_deck_list(self):
 
+to_enable = []
 for deck_id, deck in deck_store.list_enabled():
 if deck.schedule is None:
 continue
-self.schedule(RunDeck(self.director, deck_id, deck.schedule))
+to_enable.append((deck_id, deck.schedule))
+
+# If we are not initialized we should not enable anything
+if not config.is_initialized():
+to_enable = []
+
+for scheduled_task in self._scheduled_tasks[:]:
+if not isinstance(scheduled_task, RunDeck):
+continue
+
+info = (scheduled_task.deck_id, scheduled_task.schedule)
+if info in to_enable:
+# If the task is already scheduled there is no need to
+# enable it.
+to_enable.remove(info)
+else:
+# If one of the tasks that is scheduled is no longer in the
+# scheduled tasks. We should disable it.
+self.unschedule(scheduled_task)
+
+for deck_id, schedule in to_enable:
+self.schedule(RunDeck(self.director, deck_id, schedule))
 
 def _task_did_not_run(self, failure, task):
 failure.trap(DidNotRun)
diff --git a/ooni/tests/test_scheduler.py b/ooni/tests/test_scheduler.py
index cb90a30..1350dde 100644
--- a/ooni/tests/test_scheduler.py
+++ b/ooni/tests/test_scheduler.py
@@ -1,11 +1,12 @@
 import os
 import shutil
+import random
 import tempfile
 
 from twisted.internet import defer
 from twisted.trial import unittest
 
-from ooni.agent.scheduler import ScheduledTask, DidNotRun
+from ooni.agent.scheduler import ScheduledTask, DidNotRun, 
FileSystemlockAndMutex
 
 class TestScheduler(unittest.TestCase):
 def test_scheduled_task(self):
@@ -49,3 +50,28 @@ class TestScheduler(unittest.TestCase):
 
 self.assertEqual(dummy_st.should_run, False)
 shutil.rmtree(scheduler_directory)
+
+
+@defer.inlineCallbacks
+def test_filesystem_lock_and_mutex(self):
+lock_dir = tempfile.mkdtemp()
+lock_path = os.path.join(lock_dir, 'lock')
+
+lock = FileSystemlockAndMutex(lock_path)
+
+lock_count = 100
+unlock_count = 0
+dl = []
+for i in range(lock_count):
+dl.append(lock.acquire())
+if random.choice([0, 1]) == 0:
+

[tor-commits] [ooni-probe/master] Convert last runtime into UTC

[art]

commit 5d0f080b6d968172171c16cdd9a9ebd8cb887972
Author: Arturo Filastò 
Date:   Mon Sep 12 17:00:27 2016 +0200

Convert last runtime into UTC
---
 ooni/agent/scheduler.py | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index a7713e0..0a6cc0f 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -16,6 +16,8 @@ from ooni.utils.files import human_size_to_bytes, 
directory_usage
 from ooni.deck.store import input_store, deck_store, DEFAULT_DECKS
 from ooni.settings import config
 from ooni.contrib import croniter
+from ooni.contrib.dateutil.tz import tz
+
 from ooni.geoip import probe_ip
 from ooni.measurements import list_measurements
 
@@ -91,7 +93,8 @@ class ScheduledTask(object):
 return CANARY_DATE
 with self._last_run.open('r') as in_file:
 date_str = in_file.read()
-return datetime.strptime(date_str, self._time_format)
+return datetime.strptime(date_str, self._time_format).replace(
+tzinfo=tz.tzutc())
 
 def _update_last_run(self, last_run_time):
 with self._last_run.open('w') as out_file:



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Enable submission of reports when run through the web user interface

[art]

commit 2ab3219a6044b27c47dd310c1e6cecec20f006fd
Author: Arturo Filastò 
Date:   Tue Sep 13 18:57:03 2016 +0200

Enable submission of reports when run through the web user interface
---
 ooni/ui/web/server.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index aed9951..48f21c7 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -458,7 +458,7 @@ class WebUIAPI(object):
 ]
 }
 try:
-deck = NGDeck(no_collector=True)
+deck = NGDeck()
 deck.load(deck_data)
 self.run_deck(deck)
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Remove outdated comment

[art]

commit 0e452b1c70607da8d389988bb72e85b6b229fe4b
Author: Arturo Filastò 
Date:   Wed Sep 14 17:11:04 2016 +0200

Remove outdated comment
---
 ooni/resources.py | 5 -
 1 file changed, 5 deletions(-)

diff --git a/ooni/resources.py b/ooni/resources.py
index f97616c..ac0f967 100644
--- a/ooni/resources.py
+++ b/ooni/resources.py
@@ -5,11 +5,6 @@ from twisted.python.filepath import FilePath
 from twisted.internet import defer
 from twisted.web.client import downloadPage, getPage, HTTPClientFactory
 
-# WARNING: this script is being run as part of the post install procedure.
-# Be sure to not import either in this module or in the imported modules
-# dependencies other than twisted. If you end up including something that is
-# not twisted, then you will need to add it to the setup_requires in setup.py.
-
 from ooni.utils import log, gunzip, rename
 from ooni.settings import config
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [collector/master] Split up bridge descriptor tarballs into one per type.

[karsten]

commit f608c94c7f731241bf7ee8e627ca1da98c23d858
Author: Karsten Loesing 
Date:   Sun Sep 18 16:09:52 2016 +0200

Split up bridge descriptor tarballs into one per type.
---
 CHANGELOG.md  |  4 
 src/main/resources/create-tarballs.sh | 26 --
 src/main/webapp/index.html| 19 +++
 3 files changed, 31 insertions(+), 18 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e17abad..99b2918 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,10 @@
  minutes to finish properly, rather than killing them immediately.
- Replace TCP ports with hashes in @type bridge-network-status 1.1
  and @type bridge-server-descriptor 1.2.
+   - Split up bridge descriptor tarballs into one tarball per month
+ and descriptor type: bridge-statuses--MM.tar.xz,
+ bridge-server-descriptors--MM.tar.xz and
+ bridge-extra-infos--MM.tar.xz.
 
  * Minor changes
- Remove quotes around base URL in index.json.
diff --git a/src/main/resources/create-tarballs.sh 
b/src/main/resources/create-tarballs.sh
index cc75219..4f2c6eb 100755
--- a/src/main/resources/create-tarballs.sh
+++ b/src/main/resources/create-tarballs.sh
@@ -56,8 +56,12 @@ TARBALLS=(
   server-descriptors-$YEARTWO-$MONTHTWO
   extra-infos-$YEARONE-$MONTHONE
   extra-infos-$YEARTWO-$MONTHTWO
-  bridge-descriptors-$YEARONE-$MONTHONE
-  bridge-descriptors-$YEARTWO-$MONTHTWO
+  bridge-statuses-$YEARONE-$MONTHONE
+  bridge-statuses-$YEARTWO-$MONTHTWO
+  bridge-server-descriptors-$YEARONE-$MONTHONE
+  bridge-server-descriptors-$YEARTWO-$MONTHTWO
+  bridge-extra-infos-$YEARONE-$MONTHONE
+  bridge-extra-infos-$YEARTWO-$MONTHTWO
 )
 TARBALLS=($(printf "%s\n" "${TARBALLS[@]}" | uniq))
 
@@ -77,8 +81,12 @@ DIRECTORIES=(
   $OUTDIR/relay-descriptors/server-descriptor/$YEARTWO/$MONTHTWO/
   $OUTDIR/relay-descriptors/extra-info/$YEARONE/$MONTHONE/
   $OUTDIR/relay-descriptors/extra-info/$YEARTWO/$MONTHTWO/
-  $OUTDIR/bridge-descriptors/$YEARONE/$MONTHONE/
-  $OUTDIR/bridge-descriptors/$YEARTWO/$MONTHTWO/
+  $OUTDIR/bridge-descriptors/$YEARONE/$MONTHONE/statuses/
+  $OUTDIR/bridge-descriptors/$YEARTWO/$MONTHTWO/statuses/
+  $OUTDIR/bridge-descriptors/$YEARONE/$MONTHONE/server-descriptors/
+  $OUTDIR/bridge-descriptors/$YEARTWO/$MONTHTWO/server-descriptors/
+  $OUTDIR/bridge-descriptors/$YEARONE/$MONTHONE/extra-infos/
+  $OUTDIR/bridge-descriptors/$YEARTWO/$MONTHTWO/extra-infos/
 )
 DIRECTORIES=($(printf "%s\n" "${DIRECTORIES[@]}" | uniq))
 
@@ -109,8 +117,14 @@ cd $CURRENTPATH
 
 echo `date` "Finished tarball creation.  Starting symlink-update ..."
 
-mkdir -p $ARCHIVEDIR/bridge-descriptors/
-ln -f -s -t $ARCHIVEDIR/bridge-descriptors/ 
$TARBALLTARGETDIR/bridge-descriptors-20??-??.tar.xz
+mkdir -p $ARCHIVEDIR/bridge-descriptors/statuses/
+ln -f -s -t $ARCHIVEDIR/bridge-descriptors/statuses/ 
$TARBALLTARGETDIR/bridge-statuses-20??-??.tar.xz
+
+mkdir -p $ARCHIVEDIR/bridge-descriptors/server-descriptors/
+ln -f -s -t $ARCHIVEDIR/bridge-descriptors/server-descriptors/ 
$TARBALLTARGETDIR/bridge-server-descriptors-20??-??.tar.xz
+
+mkdir -p $ARCHIVEDIR/bridge-descriptors/extra-infos/
+ln -f -s -t $ARCHIVEDIR/bridge-descriptors/extra-infos/ 
$TARBALLTARGETDIR/bridge-extra-infos-20??-??.tar.xz
 
 mkdir -p $ARCHIVEDIR/bridge-pool-assignments/
 ln -f -s -t $ARCHIVEDIR/bridge-pool-assignments/ 
$TARBALLTARGETDIR/bridge-pool-assignments-20??-??.tar.xz
diff --git a/src/main/webapp/index.html b/src/main/webapp/index.html
index bbc70f2..7ed9145 100644
--- a/src/main/webapp/index.html
+++ b/src/main/webapp/index.html
@@ -168,21 +168,21 @@
   Bridge Network Statuses
   @type bridge-network-status 1.1
   recent
-  archive
+  archive
   format
 
 
   Bridge Server Descriptors
   @type bridge-server-descriptor 1.2
   recent
-  archive
+  archive
   format
 
 
   Bridge Extra-info Descriptors
   @type bridge-extra-info 1.3
   recent
-  archive
+  archive
   format
 
 
@@ -500,8 +500,6 @@ descriptor including all signatures.
 Sanitized bridge network statuses are similar to version 2 relay network
 statuses, but with only a published line in the header and
 without any lines in the footer.
-The bridge descriptor archive tarballs contain all bridge
-descriptors of a given month, not just network statuses.
 The format has changed over time to accomodate changes to the sanitizing
 process, with earlier versions being:
 
@@ -523,10 +521,8 @@ ports.
 
 Bridge server descriptors follow the same format as relay server
 descriptors, except for the sanitizing steps described above.
-The bridge descriptor archive tarballs contain all bridge
-descriptors of a given month, not just server descriptors.
-These tarballs contain one descriptor per file, whereas recently published 
bridge descriptor
-files
+The bridge server descriptor archive tarballs contain one descriptor per
+file, whereas recently published bridge server descriptor files
 contain all descriptors collected 

[tor-commits] [collector/master] Sanitize TCP ports in bridge descriptors.

[karsten]

commit ecb053899eb965c2778cf05479c26549d67f7956
Author: Karsten Loesing 
Date:   Fri Jun 10 13:28:42 2016 +0200

Sanitize TCP ports in bridge descriptors.

Implements #19317.
---
 CHANGELOG.md   |  2 +
 .../bridgedescs/SanitizedBridgesWriter.java| 81 +-
 src/main/webapp/index.html | 46 ++--
 3 files changed, 107 insertions(+), 22 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 24206ee..e17abad 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,8 @@
- Add support for Bifroest's bridge descriptor tarballs.
- Use a shutdown hook that gives currently running modules up to 10
  minutes to finish properly, rather than killing them immediately.
+   - Replace TCP ports with hashes in @type bridge-network-status 1.1
+ and @type bridge-server-descriptor 1.2.
 
  * Minor changes
- Remove quotes around base URL in index.json.
diff --git 
a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
 
b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
index b787f78..b61cd30 100644
--- 
a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
+++ 
b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
@@ -138,7 +138,8 @@ public class SanitizedBridgesWriter extends CollecTorMain {
 while ((line = br.readLine()) != null) {
   String[] parts = line.split(",");
   if ((line.length() != ("-MM,".length() + 31 * 2)
-  && line.length() != ("-MM,".length() + 50 * 2))
+  && line.length() != ("-MM,".length() + 50 * 2)
+  && line.length() != ("-MM,".length() + 83 * 2))
   || parts.length != 2) {
 logger.warn("Invalid line in bridge-ip-secrets file "
 + "starting with '" + line.substring(0, 7) + "'! "
@@ -218,8 +219,10 @@ public class SanitizedBridgesWriter extends CollecTorMain {
   scrubbedAddressPart = this.scrubIpv4Address(addressPart,
   fingerprintBytes, published);
 }
+String scrubbedPort = this.scrubTcpPort(portPart, fingerprintBytes,
+published);
 return (scrubbedAddressPart == null ? null :
-  scrubbedAddressPart + ":" + portPart);
+  scrubbedAddressPart + ":" + scrubbedPort);
   }
 
   private String scrubIpv4Address(String address, byte[] fingerprintBytes,
@@ -334,14 +337,42 @@ public class SanitizedBridgesWriter extends CollecTorMain 
{
 return sb.toString();
   }
 
+  private String scrubTcpPort(String portString, byte[] fingerprintBytes,
+  String published) throws IOException {
+if (portString.equals("0")) {
+  return "0";
+} else if (this.replaceIpAddressesWithHashes) {
+  if (this.persistenceProblemWithSecrets) {
+/* There's a persistence problem, so we shouldn't scrub more TCP
+ * ports in this execution. */
+return null;
+  }
+  byte[] hashInput = new byte[2 + 20 + 33];
+  int portNumber = Integer.parseInt(portString);
+  hashInput[0] = (byte) (portNumber >> 8);
+  hashInput[1] = (byte) portNumber;
+  System.arraycopy(fingerprintBytes, 0, hashInput, 2, 20);
+  String month = published.substring(0, "-MM".length());
+  byte[] secret = this.getSecretForMonth(month);
+  System.arraycopy(secret, 50, hashInput, 22, 33);
+  byte[] hashOutput = DigestUtils.sha256(hashInput);
+  int hashedPort = hashOutput[0] & 0xFF) << 8)
+  | (hashOutput[1] & 0xFF)) >> 2) | 0xC000;
+  return String.valueOf(hashedPort);
+} else {
+  return "1";
+}
+  }
+
   private byte[] getSecretForMonth(String month) throws IOException {
 if (!this.secretsForHashingIpAddresses.containsKey(month)
-|| this.secretsForHashingIpAddresses.get(month).length == 31) {
-  byte[] secret = new byte[50];
+|| this.secretsForHashingIpAddresses.get(month).length < 83) {
+  byte[] secret = new byte[83];
   this.secureRandom.nextBytes(secret);
   if (this.secretsForHashingIpAddresses.containsKey(month)) {
 System.arraycopy(this.secretsForHashingIpAddresses.get(month), 0,
-secret, 0, 31);
+secret, 0,
+this.secretsForHashingIpAddresses.get(month).length);
   }
   if (month.compareTo(
   this.bridgeSanitizingCutOffTimestamp) < 0) {
@@ -362,8 +393,8 @@ public class SanitizedBridgesWriter extends CollecTorMain {
   bw.close();
 } catch (IOException e) {
   logger.warn("Could not store new secret "
-  + "to disk! Not calculating any IP address hashes in "
-  + "this execution!", e);
+  + "to disk! Not calculating any IP address or TCP port "
+  + "hashes in this execution!", e);
   this.persistenceProblemWithSecrets = true;
   throw new 

[tor-commits] [ooni-probe/master] Implementing bridge failover in ooniprobe (#540)

[art]

commit 6f8fe6f7fa9197b2465bcbe3374722803f6a24d0
Author: Arturo Filastò 
Date:   Wed Jun 29 16:38:09 2016 +0200

Implementing bridge failover in ooniprobe (#540)

* Implementing bridge failover in ooniprobe

We support failing over to obfs4 and meek when vanilla tor does not work.

* This implements #538

* Reset the DataDirectory when we the data_dir is not set

Otherwise txtorcon will delete the datadirectory after it shutsdown and not
re-create it.
---
 ooni/utils/onion.py | 13 +
 1 file changed, 13 insertions(+)

diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py
index cd13640..cc2f2ff 100644
--- a/ooni/utils/onion.py
+++ b/ooni/utils/onion.py
@@ -239,6 +239,19 @@ class TorLauncherWithRetries(object):
 self.tor_config = new_tor_config
 self.timeout = timeout
 
+def _reset_tor_config(self):
+"""
+This is used to reset the Tor configuration to before launch_tor
+modified it. This is in particular used to force the regeneration of 
the
+DataDirectory.
+"""
+new_tor_config = TorConfig()
+for key in self.tor_config:
+if config.tor.data_dir is None and key == "DataDirectory":
+continue
+setattr(new_tor_config, key, getattr(self.tor_config, key))
+self.tor_config = new_tor_config
+
 def _progress_updates(self, prog, tag, summary):
 log.msg("%d%%: %s" % (prog, summary))
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Add support for starting tests from the HTTP API

[art]

commit b2118234e7b85ad4381ef2caddb9ec37e131fba4
Author: Arturo Filastò 
Date:   Fri Apr 1 19:48:14 2016 +0200

Add support for starting tests from the HTTP API
---
 ooni/director.py  |   2 +-
 ooni/web/resources.py | 172 ++
 ooni/web/root.py  |  40 
 ooni/webui.py |  57 +
 4 files changed, 270 insertions(+), 1 deletion(-)

diff --git a/ooni/director.py b/ooni/director.py
index d6503f2..9897ffd 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -134,7 +134,7 @@ class Director(object):
 elif config.tor.control_port and config.tor_state is None:
 yield connect_to_control_port()
 
-if config.global_options['no-geoip']:
+if config.global_options.get('no-geoip'):
 aux = [False]
 if config.global_options.get('annotations') is not None:
 annotations = [k.lower() for k in 
config.global_options['annotations'].keys()]
diff --git a/ooni/web/resources.py b/ooni/web/resources.py
new file mode 100644
index 000..336fdc0
--- /dev/null
+++ b/ooni/web/resources.py
@@ -0,0 +1,172 @@
+import json
+from twisted.web import resource
+from twisted.python import usage
+
+from ooni import errors
+from ooni.nettest import NetTestLoader
+
+
+class WuiResource(resource.Resource):
+isLeaf = True
+XSRF_HEADER = 'X-XSRF-TOKEN'
+# XXX set this to true when stable version
+XSRF_PROTECTION = False
+
+def __init__(self, director):
+self.director = director
+resource.Resource.__init__(self)
+
+def check_xsrf(self, request):
+if self.XSRF_PROTECTION is False:
+return True
+if request.requestHeaders.hasHeader(self.XSRF_HEADER):
+return True
+return False
+
+def render(self, request):
+if not self.check_xsrf(request):
+obj = {
+'error_code': 400,
+'error_message': ('Missing cross site request forgery '
+  'header \'{}\''.format(self.XSRF_HEADER))
+}
+request.setResponseCode(403)
+return self.render_json(obj, request)
+obj = resource.Resource.render(self, request)
+return self.render_json(obj, request)
+
+def render_json(self, obj, request):
+json_string = json.dumps(obj) + "\n"
+request.setHeader('Content-Type', 'application/json')
+request.setHeader('Content-Length', len(json_string))
+return json_string
+
+
+class DecksGenerate(WuiResource):
+def render_GET(self, request):
+return {"generate": "deck"}
+
+
+class DecksStart(WuiResource):
+def __init__(self, director, deck_name):
+WuiResource.__init__(self, director)
+self.deck_name = deck_name
+
+def render_GET(self, request):
+return {"start": self.deck_name}
+
+
+class DecksStop(WuiResource):
+def __init__(self, director, deck_id):
+WuiResource.__init__(self, director)
+self.deck_id = deck_id
+
+def render_GET(self, request):
+return {"stop": self.deck_id}
+
+
+class DecksStatus(WuiResource):
+def __init__(self, director, deck_name):
+WuiResource.__init__(self, director)
+self.deck_name = deck_name
+
+def render_GET(self, request):
+return {"deck": self.deck_name}
+
+
+class DecksList(WuiResource):
+def render_GET(self, request):
+return {"deck": "list"}
+
+
+def getNetTestLoader(test_options, test_file):
+"""
+Args:
+test_options: (dict) containing as keys the option names.
+
+test_file: (string) the path to the test_file to be run.
+Returns:
+an instance of :class:`ooni.nettest.NetTestLoader` with the specified
+test_file and the specified options.
+"""
+options = []
+for k, v in test_options.items():
+options.append('--'+k)
+options.append(v)
+
+net_test_loader = NetTestLoader(options,
+test_file=test_file)
+return net_test_loader
+
+class TestsStart(WuiResource):
+def __init__(self, director, test_name):
+WuiResource.__init__(self, director)
+self.test_name = test_name
+
+def render_POST(self, request):
+try:
+net_test = self.director.netTests[self.test_name]
+except KeyError:
+request.setResponseCode(500)
+return {
+'error_code': 500,
+'error_message': 'Could not find the specified test'
+}
+test_options = json.load(request.content)
+net_test_loader = getNetTestLoader(test_options, net_test['path'])
+try:
+net_test_loader.checkOptions()
+# XXX we actually want to generate the report_filename in a smart
+# way so that we can know where it is located and learn the results
+# of the measurement.
+report_filename 

[tor-commits] [ooni-probe/master] Add a configuration option that allows to prioritise non-onion backends

[art]

commit 2eab4fe68b0dfbb082f3253ace5321afb79777dc
Author: Arturo Filastò 
Date:   Thu Jun 16 20:03:57 2016 +0300

Add a configuration option that allows to prioritise non-onion backends

This option allows a user to specify that they wish to use either a https,
cloudfronted or onion backend server.

Write unittests to use the priority address
---
 ooni/oonicli.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index 23c0b85..2eb1d9f 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -44,9 +44,9 @@ class Options(usage.Options):
 ["collector", "c", None, "Specify the address of the collector for "
  "test results. In most cases a user will "
  "prefer to specify a bouncer over this."],
-["bouncer", "b", CANONICAL_BOUNCER_ONION, "Specify the bouncer used to 
"
-"obtain the address of the "
-"collector and test helpers."],
+["bouncer", "b", None, "Specify the bouncer used to "
+   "obtain the address of the "
+   "collector and test helpers."],
 ["logfile", "l", None, "Write to this logs to this filename."],
 ["pcapfile", "O", None, "Write a PCAP of the ooniprobe session to "
 "this filename."],



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Write the deck file to always the same filename deck/

[art]

commit 4239679d2d8d28828f6215b70678ab9c99887bd6
Author: Arturo Filastò 
Date:   Wed Jun 29 20:14:01 2016 +0200

Write the deck file to always the same filename deck/

* Add some basic unittests for oonideckgen
---
 ooni/deckgen/cli.py| 39 +++
 ooni/tests/bases.py|  1 +
 ooni/tests/test_oonideckgen.py | 35 +++
 3 files changed, 59 insertions(+), 16 deletions(-)

diff --git a/ooni/deckgen/cli.py b/ooni/deckgen/cli.py
index a6046f3..9f2cc4c 100644
--- a/ooni/deckgen/cli.py
+++ b/ooni/deckgen/cli.py
@@ -1,7 +1,10 @@
+from __future__ import print_function
+
 import os
 import sys
 import copy
 import errno
+import shutil
 
 import yaml
 
@@ -14,7 +17,6 @@ from ooni.settings import config
 
 from ooni.deckgen import __version__
 from ooni.deckgen.processors import citizenlab_test_lists
-from ooni.deckgen.processors import namebench_dns_servers
 from ooni.resources.update import download_resources
 
 class Options(usage.Options):
@@ -70,7 +72,7 @@ class Deck(object):
 self.deck_entries.append(deck_entry)
 
 def pprint(self):
-print yaml.safe_dump(self.deck_entries)
+print(yaml.safe_dump(self.deck_entries))
 
 def write_to_file(self, filename):
 with open(filename, "w+") as f:
@@ -85,8 +87,8 @@ def generate_deck(options):
 options['output']
 )
 except Exception:
-print "Could not generate country specific url list"
-print "We will just use the global one."
+print("Could not generate country specific url list")
+print("We will just use the global one.")
 
 url_list_global = citizenlab_test_lists.generate_global_input(
 options['output']
@@ -105,9 +107,9 @@ def generate_deck(options):
 deck_filename = os.path.join(options['output'],
  "default-user.deck")
 deck.write_to_file(deck_filename)
-print "Deck written to %s" % deck_filename
-print "Run ooniprobe like so:"
-print "ooniprobe -i %s" % deck_filename
+print("Deck written to %s" % deck_filename)
+print("Run ooniprobe like so:")
+print("ooniprobe -i %s" % deck_filename)
 
 
 @defer.inlineCallbacks
@@ -138,8 +140,8 @@ def run():
 try:
 options.parseOptions()
 except usage.UsageError as error_message:
-print "%s: %s" % (sys.argv[0], error_message)
-print options
+print("%s: %s" % (sys.argv[0], error_message))
+print(options)
 sys.exit(1)
 
 if not resources_up_to_date():
@@ -154,24 +156,29 @@ def run():
 try:
 options['country-code'] = yield get_user_country_code()
 except errors.ProbeIPUnknown:
-print "Could not determine your IP address."
-print "Check your internet connection or specify a country code 
with -c."
+print("Could not determine your IP address.")
+print("Check your internet connection or specify a country code "
+  "with -c.")
 sys.exit(4)
 
 if len(options['country-code']) != 2:
-print "%s: --country-code must be 2 characters" % sys.argv[0]
+print("%s: --country-code must be 2 characters" % sys.argv[0])
 sys.exit(2)
 
 if not os.path.isdir(options['output']):
-print "%s: %s is not a directory" % (sys.argv[0],
- options['output'])
+print("%s: %s is not a directory" % (sys.argv[0],
+ options['output']))
 sys.exit(3)
 
 options['country-code'] = options['country-code'].lower()
 
 output_dir = os.path.abspath(options['output'])
-output_dir = os.path.join(output_dir,
-  "deck-%s" % options['country-code'])
+output_dir = os.path.join(output_dir, "deck")
+
+if os.path.isdir(output_dir):
+print("Found previous deck deleting content of it")
+shutil.rmtree(output_dir)
+
 options['output'] = output_dir
 
 try:
diff --git a/ooni/tests/bases.py b/ooni/tests/bases.py
index 31cbf94..904aaba 100644
--- a/ooni/tests/bases.py
+++ b/ooni/tests/bases.py
@@ -1,5 +1,6 @@
 import os
 import shutil
+
 from twisted.trial import unittest
 
 from ooni.settings import config
diff --git a/ooni/tests/test_oonideckgen.py b/ooni/tests/test_oonideckgen.py
new file mode 100644
index 000..4a52377
--- /dev/null
+++ b/ooni/tests/test_oonideckgen.py
@@ -0,0 +1,35 @@
+import os
+import yaml
+import tempfile
+
+from .bases import ConfigTestCase
+
+from ooni.deckgen import cli
+
+class TestOONIDeckgen(ConfigTestCase):
+def setUp(self):
+super(TestOONIDeckgen, self).setUp()
+test_lists_dir = os.path.join(self.config.ooni_home,
+  "resources",
+  "citizenlab-test-lists")
+try:
+os.makedirs(test_lists_dir)
+

[tor-commits] [ooni-probe/master] Implementing bridge failover in ooniprobe

[art]

commit 90ec4fb643841b4a2c97fa2985e66d7138049467
Author: Arturo Filastò 
Date:   Tue Jun 14 16:15:44 2016 +0300

Implementing bridge failover in ooniprobe

We support failing over to obfs4 and meek when vanilla tor does not work.

* This implements #538
---
 ooni/oonicli.py | 6 +++---
 ooni/utils/onion.py | 1 +
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index 2eb1d9f..23c0b85 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -44,9 +44,9 @@ class Options(usage.Options):
 ["collector", "c", None, "Specify the address of the collector for "
  "test results. In most cases a user will "
  "prefer to specify a bouncer over this."],
-["bouncer", "b", None, "Specify the bouncer used to "
-   "obtain the address of the "
-   "collector and test helpers."],
+["bouncer", "b", CANONICAL_BOUNCER_ONION, "Specify the bouncer used to 
"
+"obtain the address of the "
+"collector and test helpers."],
 ["logfile", "l", None, "Write to this logs to this filename."],
 ["pcapfile", "O", None, "Write a PCAP of the ooniprobe session to "
 "this filename."],
diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py
index e18a6ee..cd13640 100644
--- a/ooni/utils/onion.py
+++ b/ooni/utils/onion.py
@@ -237,6 +237,7 @@ class TorLauncherWithRetries(object):
 continue
 setattr(new_tor_config, key, getattr(self.tor_config, key))
 self.tor_config = new_tor_config
+self.timeout = timeout
 
 def _progress_updates(self, prog, tag, summary):
 log.msg("%d%%: %s" % (prog, summary))



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Add backend to required options to show error when it's not provided

[art]

commit 2b688756d9417d1c7279e8b4054fd739e5ca5c35
Author: Arturo Filastò 
Date:   Sun Jul 24 23:46:36 2016 +0200

Add backend to required options to show error when it's not provided

* Use same function to represent packets in traceroute test
---
 ooni/nettests/manipulation/traceroute.py |  5 +++--
 ooni/templates/scapyt.py | 10 +-
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/ooni/nettests/manipulation/traceroute.py 
b/ooni/nettests/manipulation/traceroute.py
index d6f1ddf..3553bfd 100644
--- a/ooni/nettests/manipulation/traceroute.py
+++ b/ooni/nettests/manipulation/traceroute.py
@@ -31,6 +31,7 @@ class Traceroute(scapyt.BaseScapyTest):
   "set to 0, 22, 23, 53, 80, 123, 443, 8080 and 65535."
 
 requiredTestHelpers = {'backend': 'traceroute'}
+requiredOptions = ['backend']
 requiresRoot = True
 requiresTor = False
 
@@ -63,9 +64,9 @@ class Traceroute(scapyt.BaseScapyTest):
 st.stopListening()
 st.matchResponses()
 for packet in st.sent_packets:
-self.report['sent_packets'].append(packet)
+self.report['sent_packets'].append(scapyt.representPacket(packet))
 for packet in st.matched_packets.values():
-self.report['answered_packets'].extend(packet)
+
self.report['answered_packets'].append(scapyt.representPacket(packet))
 
 for ttl in xrange(st.ttl_min, st.ttl_max):
 matchedPackets = filter(
diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py
index 1c726d8..e122525 100644
--- a/ooni/templates/scapyt.py
+++ b/ooni/templates/scapyt.py
@@ -7,13 +7,13 @@ from ooni.utils.net import hasRawSocketPermission
 from ooni.utils.txscapy import ScapySender, ScapyFactory
 
 
-def _representPacket(packet):
+def representPacket(packet):
 return {
 "raw_packet": {
 'data': b64encode(str(packet)),
 'format': 'base64'
 },
-"summary": repr(packet)
+"summary": str(repr(packet))
 }
 
 class BaseScapyTest(NetTestCase):
@@ -98,8 +98,8 @@ class BaseScapyTest(NetTestCase):
 sent_packet.src = '127.0.0.1'
 received_packet.dst = '127.0.0.1'
 
-self.report['sent_packets'].append(_representPacket(sent_packet))
-
self.report['answered_packets'].append(_representPacket(received_packet))
+self.report['sent_packets'].append(representPacket(sent_packet))
+
self.report['answered_packets'].append(representPacket(received_packet))
 return packets
 
 def sr(self, packets, timeout=None, *arg, **kw):
@@ -152,7 +152,7 @@ class BaseScapyTest(NetTestCase):
 
 scapySender.stopSending()
 for sent_packet in packets:
-self.report['sent_packets'].append(_representPacket(sent_packet))
+self.report['sent_packets'].append(representPacket(sent_packet))
 
 
 ScapyTest = BaseScapyTest



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Merge pull request #574 from TheTorProject/hotfix/traceroute

[art]

commit 5e86276327953e5f054254b6a3f3bd7e1342bbf1
Merge: da2b173 2b68875
Author: Arturo Filastò 
Date:   Tue Jul 26 02:09:06 2016 +0200

Merge pull request #574 from TheTorProject/hotfix/traceroute

Add backend to required options to show error when it's not provided

 ooni/nettests/manipulation/traceroute.py |  5 +++--
 ooni/templates/scapyt.py | 10 +-
 2 files changed, 8 insertions(+), 7 deletions(-)



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Use the ConfigTestCase also for test_nettest

[art]

commit 0842d2158ab879e34957168f509c6dfad9c3cb7a
Author: Arturo Filastò 
Date:   Wed Jun 29 20:50:40 2016 +0200

Use the ConfigTestCase also for test_nettest
---
 ooni/tests/test_nettest.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ooni/tests/test_nettest.py b/ooni/tests/test_nettest.py
index 4298fe9..239080a 100644
--- a/ooni/tests/test_nettest.py
+++ b/ooni/tests/test_nettest.py
@@ -223,6 +223,7 @@ class TestNetTest(ConfigTestCase):
 timeout = 1
 
 def setUp(self):
+super(TestNetTest, self).setUp()
 self.filename = ""
 with open(dummyInputFile, 'w') as f:
 for i in range(10):
@@ -230,6 +231,7 @@ class TestNetTest(ConfigTestCase):
 super(TestNetTest, self).setUp()
 
 def tearDown(self):
+super(TestNetTest, self).tearDown()
 os.remove(dummyInputFile)
 if self.filename != "":
 os.remove(self.filename)



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Bump version number of master

[art]

commit 563e4ded184808ab68b36b3043e5a3ba7d6195f7
Author: Arturo Filastò 
Date:   Fri Jul 29 13:42:18 2016 +0200

Bump version number of master
---
 ooni/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ooni/__init__.py b/ooni/__init__.py
index 8185030..9d99323 100644
--- a/ooni/__init__.py
+++ b/ooni/__init__.py
@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 
 __author__ = "Open Observatory of Network Interference"
-__version__ = "1.6.1"
+__version__ = "1.6.2.dev0"
 # This is the version number of resources to be downloaded
 # when a release is made it should be aligned to __version__
 __resources_version__ = "1.6.0"



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Delete inputs, resources and decks data directories

[art]

commit 9b48ff3033e93e7386a70f8649657e70abe061cf
Author: Arturo Filastò 
Date:   Tue Jul 26 16:14:51 2016 +0200

Delete inputs, resources and decks data directories
---
 MANIFEST.in  |  2 --
 data/decks/complete.deck | 72 
 data/decks/complete_no_root.deck | 60 -
 data/decks/fast.deck | 36 
 data/decks/fast_no_root.deck | 24 --
 data/decks/mlab.deck | 12 ---
 data/inputs/.gitignore   |  2 --
 data/resources/README|  2 --
 8 files changed, 210 deletions(-)

diff --git a/MANIFEST.in b/MANIFEST.in
index e2903fb..692c73f 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,6 +1,4 @@
 include README.rst ChangeLog.rst requirements.txt LICENSE
-recursive-include data/decks *
-recursive-include data/inputs *
 include data/oonideckgen.1
 include data/ooniprobe.1
 include data/oonireport.1
diff --git a/data/decks/complete.deck b/data/decks/complete.deck
deleted file mode 100644
index 7d9346c..000
--- a/data/decks/complete.deck
+++ /dev/null
@@ -1,72 +0,0 @@
-- options:
-collector: null
-help: 0
-logfile: null
-no-default-reporter: 0
-parallelism: null
-pcapfile: null
-reportfile: null
-resume: 0
-subargs: [-f, 
'httpo://ihiderha53f36lsd.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1']
-test_file: blocking/http_requests
-testdeck: null
-- options:
-collector: null
-help: 0
-logfile: null
-no-default-reporter: 0
-parallelism: null
-pcapfile: null
-reportfile: null
-resume: 0
-subargs: [-f, 
'httpo://ihiderha53f36lsd.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1']
-test_file: blocking/dns_consistency
-testdeck: null
-- options:
-collector: null
-help: 0
-logfile: null
-no-default-reporter: 0
-parallelism: null
-pcapfile: null
-reportfile: null
-resume: 0
-subargs: []
-test_file: manipulation/http_invalid_request_line
-testdeck: null
-- options:
-collector: null
-help: 0
-logfile: null
-no-default-reporter: 0
-parallelism: null
-pcapfile: null
-reportfile: null
-resume: 0
-subargs: []
-test_file: manipulation/http_header_field_manipulation
-testdeck: null
-- options:
-collector: null
-help: 0
-logfile: null
-no-default-reporter: 0
-parallelism: null
-pcapfile: null
-reportfile: null
-resume: 0
-subargs: []
-test_file: manipulation/traceroute
-testdeck: null
-- options:
-collector: null
-help: 0
-logfile: null
-no-default-reporter: 0
-parallelism: null
-pcapfile: null
-reportfile: null
-resume: 0
-subargs: [-f, 
'httpo://ihiderha53f36lsd.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1']
-test_file: manipulation/http_host
-testdeck: null
diff --git a/data/decks/complete_no_root.deck b/data/decks/complete_no_root.deck
deleted file mode 100644
index ba2c29b..000
--- a/data/decks/complete_no_root.deck
+++ /dev/null
@@ -1,60 +0,0 @@
-- options:
-collector: null
-help: 0
-logfile: null
-no-default-reporter: 0
-parallelism: null
-pcapfile: null
-reportfile: null
-resume: 0
-subargs: [-f, 
'httpo://ihiderha53f36lsd.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1']
-test_file: blocking/http_requests
-testdeck: null
-- options:
-collector: null
-help: 0
-logfile: null
-no-default-reporter: 0
-parallelism: null
-pcapfile: null
-reportfile: null
-resume: 0
-subargs: [-f, 
'httpo://ihiderha53f36lsd.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1']
-test_file: blocking/dns_consistency
-testdeck: null
-- options:
-collector: null
-help: 0
-logfile: null
-no-default-reporter: 0
-parallelism: null
-pcapfile: null
-reportfile: null
-resume: 0
-subargs: []
-test_file: manipulation/http_invalid_request_line
-testdeck: null
-- options:
-collector: null
-help: 0
-logfile: null
-no-default-reporter: 0
-parallelism: null
-pcapfile: null
-reportfile: null
-resume: 0
-subargs: []
-test_file: manipulation/http_header_field_manipulation
-testdeck: null
-- options:
-collector: null
-help: 0
-logfile: null
-no-default-reporter: 0
-parallelism: null
-pcapfile: null
-reportfile: null
-resume: 0
-subargs: [-f, 
'httpo://ihiderha53f36lsd.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1']
-test_file: manipulation/http_host
-testdeck: null
diff --git a/data/decks/fast.deck b/data/decks/fast.deck
deleted file mode 100644
index ddf3c71..000
--- a/data/decks/fast.deck
+++ /dev/null
@@ -1,36 +0,0 @@

[tor-commits] [ooni-probe/master] Delete dead code

[art]

commit 220c699d52620a4dee887ccc8438548df10960cd
Author: Arturo Filastò 
Date:   Mon Jul 25 16:07:08 2016 +0200

Delete dead code
---
 ooni/nodes.conf | 53 -
 ooni/oonid.py   | 15 ---
 2 files changed, 68 deletions(-)

diff --git a/ooni/nodes.conf b/ooni/nodes.conf
deleted file mode 100644
index 4d14be1..000
--- a/ooni/nodes.conf
+++ /dev/null
@@ -1,53 +0,0 @@
-[control]
-# Specify in the list of network nodes to
-# be used as control nodes
-[[node1]]
-address = socks4a://user:password@192.0.2.1
-
-[[node2]]
-address = socks5://user:password@192.0.2.2
-
-[[node3]]
-address = ssh://user:password@192.0.2.3
-
-
-[experiment]
-# Specify the list of experiment nodes to
-# be used for running tests.
-[[countryA]]
-address = socks4a://user:password@192.0.2.1
-family = china
-
-[[countryA1]]
-address = socks5://user:password@192.0.2.2
-family = china
-
-[[countryA2]]
-address = ssh://user:password@192.0.2.3
-family = italy
-
-[operators]
-# Specify the list of operator nodes. These are
-# the ones capable of accepting bundles of units
-# of work and scheduling work to experiment nodes.
-[[Local box]]
-address = localhost
-capacity = 0.2
-
-[[Planetlab node]]
-address = ssh://user:password@192.0.2.3
-rsakey = ~/.ssh/id_rsa
-other =
-# Specify an float between 0 and 1 to how much
-# work this operator node should support.
-capacity = 0.4
-
-[[M-Lab server]]
-address = ssh://user:password@192.0.2.3
-capacity = 0.5
-
-
-
-
-
-
diff --git a/ooni/oonid.py b/ooni/oonid.py
deleted file mode 100644
index c04cadc..000
--- a/ooni/oonid.py
+++ /dev/null
@@ -1,15 +0,0 @@
-from twisted.application import service, internet
-
-from ooni.settings import config
-from ooni.api.spec import oonidApplication
-from ooni.director import Director
-
-def getOonid():
-director = Director()
-director.start()
-oonidApplication.director = director
-return internet.TCPServer(int(config.advanced.oonid_api_port), 
oonidApplication)
-
-application = service.Application("ooniprobe")
-service = getOonid()
-service.setServiceParent(application)



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] ignoreExistingDirectory is a recent twisted thing.

[art]

commit 21e6b839918e88f8260a58b23614194ec1230247
Author: Arturo Filastò 
Date:   Fri Aug 5 17:16:50 2016 +0200

ignoreExistingDirectory is a recent twisted thing.

Wrap calls to makedirs with proper exception handling.
---
 ooni/deck/store.py | 13 +++--
 ooni/resources.py  | 13 +++--
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/ooni/deck/store.py b/ooni/deck/store.py
index 2d24f29..295f817 100644
--- a/ooni/deck/store.py
+++ b/ooni/deck/store.py
@@ -1,5 +1,6 @@
 import csv
 import json
+import errno
 from copy import deepcopy
 
 from twisted.internet import defer
@@ -76,8 +77,16 @@ class InputStore(object):
 self.path = FilePath(config.inputs_directory)
 self.resources = FilePath(config.resources_directory)
 
-self.path.child("descriptors").makedirs(ignoreExistingDirectory=True)
-self.path.child("data").makedirs(ignoreExistingDirectory=True)
+try:
+self.path.child("descriptors").makedirs()
+except OSError as e:
+if not e.errno == errno.EEXIST:
+raise
+try:
+self.path.child("data").makedirs()
+except OSError as e:
+if not e.errno == errno.EEXIST:
+raise
 yield self.update_url_lists(country_code)
 
 @defer.inlineCallbacks
diff --git a/ooni/resources.py b/ooni/resources.py
index 9615c53..f97616c 100644
--- a/ooni/resources.py
+++ b/ooni/resources.py
@@ -1,4 +1,5 @@
 import json
+import errno
 
 from twisted.python.filepath import FilePath
 from twisted.internet import defer
@@ -107,7 +108,11 @@ def check_for_update(country_code=None):
 latest_version = yield get_latest_version()
 
 resources_dir = FilePath(config.resources_directory)
-resources_dir.makedirs(ignoreExistingDirectory=True)
+try:
+resources_dir.makedirs()
+except OSError as e:
+if not e.errno == errno.EEXIST:
+raise
 current_manifest = resources_dir.child("manifest.json")
 
 if current_manifest.exists():
@@ -149,7 +154,11 @@ def check_for_update(country_code=None):
 filename = filename[:-3]
 gzipped = True
 dst_file = resources_dir.child(pre_path).child(filename)
-dst_file.parent().makedirs(ignoreExistingDirectory=True)
+try:
+dst_file.parent().makedirs()
+except OSError as e:
+if not e.errno == errno.EEXIST:
+raise
 src_file = dst_file.temporarySibling()
 src_file.alwaysCreate = 0
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] web-full.yaml: add missing word

[art]

commit eb7971a108201bed8f6735a14f7372ab9960
Author: Simone Basso 
Date:   Mon Aug 8 18:46:27 2016 +0200

web-full.yaml: add missing word
---
 data/decks/web-full.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/data/decks/web-full.yaml b/data/decks/web-full.yaml
index 7812505..ccb470d 100644
--- a/data/decks/web-full.yaml
+++ b/data/decks/web-full.yaml
@@ -1,7 +1,7 @@
 ---
 name: Full Web test deck
 description: This deck runs HTTP Header Field Manipulation, HTTP Invalid
-Request and the Web Connectivity test
+Request Line and the Web Connectivity test
 schedule: "@daily"
 tasks:
 - name: Runs the HTTP Header Field Manipulation test



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Generate the filename for the report late on to avoid conflicts in measurement IDs

[art]

commit 40b83a84265a24bd6777195b43a36edc8dc42640
Author: Arturo Filastò 
Date:   Tue Sep 6 22:05:15 2016 +0200

Generate the filename for the report late on to avoid conflicts in 
measurement IDs
---
 ooni/deck/deck.py | 18 ++
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/ooni/deck/deck.py b/ooni/deck/deck.py
index e4c35c4..3fc10ab 100644
--- a/ooni/deck/deck.py
+++ b/ooni/deck/deck.py
@@ -1,4 +1,6 @@
 import os
+import uuid
+import errno
 from copy import deepcopy
 from string import Template
 
@@ -90,7 +92,6 @@ class NGDeck(object):
 
 self._measurement_path = FilePath(config.measurements_directory)
 self._tasks = []
-self.task_ids = []
 
 if deck_path is not None:
 self.open(deck_path)
@@ -145,7 +146,6 @@ class NGDeck(object):
 self.bouncer.backend_type == "onion"):
 self.requires_tor = True
 self._tasks.append(deck_task)
-self.task_ids.append(deck_task.id)
 
 if self.metadata.get('no_collector', False):
 self.no_collector = True
@@ -217,7 +217,10 @@ class NGDeck(object):
 
 def _run_ooni_task(self, task, director):
 net_test_loader = task.ooni["net_test_loader"]
-test_details = task.ooni["test_details"]
+# XXX-REFACTOR we do this so late to avoid the collision between the
+#  same id and hence generating the same filename.
+test_details = net_test_loader.getTestDetails()
+task.id = generate_filename(test_details)
 
 measurement_id = None
 report_filename = task.output_path
@@ -228,9 +231,9 @@ class NGDeck(object):
 try:
 measurement_dir.createDirectory()
 except OSError as ose:
-# Ignore 'File Exists'
-if ose.errno != 17:
-raise
+if ose.errno == errno.EEXIST:
+raise Exception("Directory already exists, there is a "
+"collision")
 
 report_filename = 
measurement_dir.child("measurements.njson.progress").path
 pid_file = measurement_dir.child("running.pid")
@@ -388,10 +391,9 @@ class DeckTask(object):
 )
 file_path = resolve_file_path(filename, self.cwd)
 input_file['test_options'][input_file['key']] = file_path
-self.ooni['test_details'] = 
self.ooni['net_test_loader'].getTestDetails()
-self.id = generate_filename(self.ooni['test_details'])
 
 def setup(self):
+self.id = str(uuid.uuid4())
 return getattr(self, "_setup_"+self.type)()
 
 def _load(self, data):



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Check for all the possible pid file locations when checking status of ooniprobe-agent

[art]

commit bb7bcdd18d6a94097cf4de057c752828cfdf16c6
Author: Arturo Filastò 
Date:   Thu Sep 1 17:29:23 2016 +0200

Check for all the possible pid file locations when checking status of 
ooniprobe-agent

* This fixes https://github.com/TheTorProject/ooni-probe/issues/597
---
 ooni/scripts/ooniprobe_agent.py | 41 -
 ooni/settings.py|  8 
 setup.py| 14 ++
 3 files changed, 38 insertions(+), 25 deletions(-)

diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py
index 08fba89..ecfe7d8 100644
--- a/ooni/scripts/ooniprobe_agent.py
+++ b/ooni/scripts/ooniprobe_agent.py
@@ -67,6 +67,11 @@ def start_agent(options=None):
 twistd_config.loadedPlugins = {
 "StartOoniprobeAgent": StartOoniprobeAgentPlugin()
 }
+
+if status_agent() == 0:
+print("Stop ooniprobe-agent before attempting to start it")
+return 1
+
 print("Starting ooniprobe agent.")
 WEB_UI_URL = "http://{0}:{1}".format(
 config.advanced.webui_address, config.advanced.webui_port)
@@ -76,23 +81,25 @@ def start_agent(options=None):
 return 0
 
 def status_agent():
-pidfile = os.path.join(
-config.running_path,
-'twistd.pid'
-)
-if not os.path.exists(pidfile):
-print("ooniprobe-agent is NOT running")
-return  1
-pid = open(pidfile, "r").read()
-pid = int(pid)
-try:
-os.kill(pid, signal.SIG_DFL)
-except OSError, oserr:
-if oserr.errno == 3:
-print("ooniprobe-agent is NOT running")
-return 1
-print("ooniprobe-agent is running")
-return 0
+running = False
+for pidfile in [config.system_pid_path, config.user_pid_path]:
+if not os.path.exists(pidfile):
+# Didn't find the pid_file
+continue
+pid = open(pidfile, "r").read()
+pid = int(pid)
+try:
+os.kill(pid, signal.SIG_DFL)
+running = True
+except OSError, oserr:
+if oserr.errno == 3:
+# Found pid, but isn't running
+continue
+if running is True:
+print("ooniprobe-agent is running")
+return 0
+print("ooniprobe-agent is NOT running")
+return 1
 
 def stop_agent():
 # This function is borrowed from tahoe
diff --git a/ooni/settings.py b/ooni/settings.py
index c36f3f7..065252b 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -235,6 +235,14 @@ class OConfig(object):
 return self.ooni_home
 
 @property
+def user_pid_path(self):
+return os.path.join(self.ooni_home, "twistd.pid")
+
+@property
+def system_pid_path(self):
+return os.path.join(VAR_LIB_PATH, "twistd.pid")
+
+@property
 def data_directory_candidates(self):
 dirs = [
 self.ooni_home,
diff --git a/setup.py b/setup.py
index 7612b39..e5dd836 100644
--- a/setup.py
+++ b/setup.py
@@ -69,19 +69,17 @@ When you got them run:
 Using ooniprobe
 ---
 
-To generate a test deck for your country, cd to the directory where you want it
-and run:
+It is recommended that you start the ooniprobe-agent system daemon that will
+expose a localhost only Web UI and automatically run tests for you.
 
-.. code:: bash
-
-oonideckgen
+This can be done with:
 
+.. code:: bash
 
-To setup a daily cronjob run this:
+ooniprobe-agent start
 
-.. code:: bash
 
-(crontab -l 2>/dev/null; echo "@daily ooniprobe `oonideckgen | grep -e 
'^ooniprobe'`") | crontab -
+Then connect to the local web interface on http://127.0.0.1:8842/
 
 Have fun!
 """



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Add deck configuration step to the setup wizard

[art]

commit 0c27e0602cfcfb172bc09d1ca2c35a612803f783
Author: Arturo Filastò 
Date:   Wed Aug 31 16:32:38 2016 +0200

Add deck configuration step to the setup wizard
---
 ooni/ui/web/server.py | 38 +-
 1 file changed, 37 insertions(+), 1 deletion(-)

diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index 6f9ef69..2e89370 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -190,7 +190,8 @@ class WebUIAPI(object):
 def status(self):
 quota_warning = None
 try:
-with open(os.path.join(config.running_dir, "quota_warning")) as 
in_file:
+with open(os.path.join(config.running_path,
+   "quota_warning")) as in_file:
 quota_warning = in_file.read()
 except IOError as ioe:
 if ioe.errno != errno.ENOENT:
@@ -259,6 +260,21 @@ class WebUIAPI(object):
 d.addCallback(got_status_update)
 return d
 
+@app.route('/api/initialize', methods=["GET"])
+@xsrf_protect(check=False)
+@requires_false(attrs=['_is_initialized'])
+def api_initialize_get(self, request):
+available_decks = []
+for deck_id, deck in self.director.deck_store.list():
+available_decks.append({
+'name': deck.name,
+'description': deck.description,
+'schedule': deck.schedule,
+'enabled': self.director.deck_store.is_enabled(deck_id),
+'id': deck_id
+})
+return self.render_json({"available_decks": available_decks}, request)
+
 @app.route('/api/initialize', methods=["POST"])
 @xsrf_protect(check=True)
 @requires_false(attrs=['_is_initialized'])
@@ -278,6 +294,26 @@ class WebUIAPI(object):
 raise WebUIError(400, 'Missing required key {0}'.format(
 required_key))
 config.create_config_file(**options)
+try:
+deck_config = initial_configuration['deck_config']
+except KeyError:
+raise WebUIError(400, 'Missing enabled decks')
+
+for deck_id, enabled in deck_config.items():
+try:
+if enabled is True:
+self.director.deck_store.enable(deck_id)
+elif enabled is False:
+try:
+self.director.deck_store.disable(deck_id)
+except DeckNotFound:
+# We ignore these errors, because it could be that a 
deck
+# that is marked as disabled is already disabled
+pass
+except DeckNotFound:
+raise WebUIError(404, 'Deck not found')
+
+self.scheduler.refresh_deck_list()
 config.set_initialized()
 
 self._is_initialized = True



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Add informed consent form to be used in the CLI version of ooniprobe

[art]

commit c23c47b4a314329b6bf9a778027b421801e55b82
Author: Arturo Filastò 
Date:   Thu Aug 4 19:47:15 2016 +0200

Add informed consent form to be used in the CLI version of ooniprobe
---
 ooni/ui/consent-form.md | 265 
 1 file changed, 265 insertions(+)

diff --git a/ooni/ui/consent-form.md b/ooni/ui/consent-form.md
new file mode 100644
index 000..08de2be
--- /dev/null
+++ b/ooni/ui/consent-form.md
@@ -0,0 +1,265 @@
+The [Open Observatory of Network Interference
+(OONI)](https://ooni.torproject.org/) is a free software project, under the 
[Tor
+Project](https://www.torproject.org/), which collects and processes network
+measurements with the aim of detecting network anomalies, such as censorship 
and
+traffic manipulation.
+
+Running OONI may be against the terms of service of your ISP or legally
+questionable in your country. By running OONI you will connect to web services
+which may be banned, and use web censorship circumvention methods such as Tor.
+The OONI project will publish data submitted by probes, possibly including your
+IP address or other identifying information. In addition, your use of OONI will
+be clear to anybody who has access to your computer, and to anybody who can
+monitor your internet connection (such as your employer, ISP or government).
+
+By running ooniprobe, you are participating as a volunteer in this project. 
This
+form includes information that you should be aware of and consent to *prior* to
+running ooniprobe.
+
+## OONI software tests
+
+The OONI project has developed multiple free software tests which are designed 
to:
+
+* Detect the blocking of websites
+
+* Detect systems responsible for censorship and traffic manipulation
+
+* Evaluate the reachability of [Tor bridges](https://bridges.torproject.org/),
+  proxies, VPNs, and sensitive domains
+
+Below we provide brief descriptions of how these tests work. 
+
+## Test descriptions
+
+The recommended set of tests that users run through the
+`oonideckgen` command include the following:
+
+**Web connectivity:** This test examines whether websites are reachable and if
+they are not, it attempts to determine whether access to them is blocked 
through
+DNS tampering, TCP connection RST/IP blocking or by having a transparent HTTP
+proxy. It does so by identifying the resolver of the user, performing a DNS
+lookup, attempting to establish a TCP session and by sending HTTP GET requests
+to the servers that are hosting tested websites.
+
+**HTTP invalid request line:** This test tries to detect the presence of 
network
+components (“middle box”) which could be responsible for censorship and/or
+traffic manipulation. Instead of sending a normal HTTP request, this test sends
+an invalid HTTP request line - containing an invalid HTTP version number, an
+invalid field count and a huge request method – to an echo service listening 
on
+the standard HTTP port. If a middle box is present in the tested network, the
+invalid HTTP request line will be intercepted by the middle box and this may
+trigger error messages which can help identify the proxy technologies.
+
+**HTTP header field manipulation:** This test tries to detect the presence of
+network components (“middle box”) which could be responsible for censorship
+and/or traffic manipulation. It does so by sending HTTP requests which include
+valid, but non-canonical HTTP headers to a backend control server which sends
+back any data it receives. If we receive the HTTP headers exactly as we sent
+them, then we assume that there is no “middle box” in the network. If,
+however, such software is present in the network that we are testing, it will
+likely normalize the invalid headers that we are sending or add extra headers.
+
+Another test which attempts to detect traffic manipulation includes **Multi-
+protocol traceroute**, which constructs packets in such a way that they perform
+a traceroute from multiple protocols and ports simultaneously. Other tests
+include **Tor bridge reachability**, **Psiphon**, **Lantern**, **OpenVPN** and
+**Meek fronted requests**, which examine whether these services work within a
+tested network by attempting to connect to them in an automated way.
+
+Further test descriptions can be found here.
+
+## Risks
+
+Many countries have a lengthy history of subjecting digital rights activists to
+various forms of abuse that could make it dangerous for individuals in these
+countries to run OONI. The use of OONI might therefore subject users to severe
+civil, criminal, or extra-judicial penalties, and such sanctions can 
potentially
+include:
+
+* Imprisonment
+
+* Physical assaults
+
+* Large fines
+
+* Receiving threats
+
+* Being placed on government watch lists
+
+* Targeted for surveillance
+
+While most countries don't have laws which specifically prohibit the use of
+network measurement software, it's important to note that the use of OONI can
+*still* 

[tor-commits] [ooni-probe/master] Add simple unittest for api_status

[art]

commit 417e64893fa21a9ca1b9e66efa812d784a128731
Author: Arturo Filastò 
Date:   Tue Sep 6 16:08:27 2016 +0200

Add simple unittest for api_status
---
 .coveragerc   |  2 +-
 ooni/tests/test_wui_server.py | 32 
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/.coveragerc b/.coveragerc
index 34ed07c..4f5cfba 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -1,4 +1,4 @@
 [run]
 source = ooni
 [report]
-omit = ooni/nettests/*, ooni/api/*, ooni/kit/*
+omit = ooni/nettests/*, ooni/api/*, ooni/kit/*, ooni/contrib/*
diff --git a/ooni/tests/test_wui_server.py b/ooni/tests/test_wui_server.py
new file mode 100644
index 000..ebaaacd
--- /dev/null
+++ b/ooni/tests/test_wui_server.py
@@ -0,0 +1,32 @@
+from __future__ import print_function
+
+import json
+
+from .bases import ConfigTestCase
+from ooni.ui.web.server import WebUIAPI
+
+from mock import patch, Mock
+
+class TestWebUIServer(ConfigTestCase):
+def setUp(self):
+super(TestWebUIServer, self).setUp()
+
+director = Mock()
+scheduler = Mock()
+
+self.wui_api = WebUIAPI(self.config, director, scheduler)
+
+def tearDown(self):
+super(TestWebUIServer, self).tearDown()
+self.wui_api.status_poller.stop()
+self.wui_api.director_event_poller.stop()
+
+def test_api_status(self):
+request = Mock()
+resp = self.wui_api.api_status(request)
+j = json.loads(resp)
+expected_keys = ["software_version", "software_name", "asn",
+ "country_code", "director_started", "initialized",
+ "quota_warning"]
+for key in expected_keys:
+self.assertTrue(key in j)



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Add support for initialization of ooniprobe

[art]

commit 27a299c5c7d195860e63aa0b1d316f3255e4857d
Author: Arturo Filastò 
Date:   Wed Aug 3 13:05:31 2016 +0200

Add support for initialization of ooniprobe
---
 data/decks/web.yaml   |   1 +
 ooni/agent/scheduler.py   |  25 +++---
 ooni/scripts/ooniprobe.py |   4 +-
 ooni/settings.py  | 191 ++
 ooni/tests/__init__.py|   7 --
 ooni/ui/cli.py|  52 +++-
 ooni/ui/web/client/index.html |   2 +-
 ooni/ui/web/server.py |  82 +-
 ooni/utils/__init__.py|   1 -
 9 files changed, 329 insertions(+), 36 deletions(-)

diff --git a/data/decks/web.yaml b/data/decks/web.yaml
index a81b8f8..c7b9bdc 100644
--- a/data/decks/web.yaml
+++ b/data/decks/web.yaml
@@ -2,6 +2,7 @@
 name: Web related ooniprobe tests
 description: This deck runs HTTP Header Field Manipulation, HTTP Invalid
 Request and the Web Connectivity test
+schedule: "@daily"
 tasks:
 - name: Runs the HTTP Header Field Manipulation test
   ooni:
diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index a6d689f..1f51bd4 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -22,9 +22,11 @@ class ScheduledTask(object):
 schedule = None
 identifier = None
 
-def __init__(self, schedule=None):
+def __init__(self, schedule=None, identifier=None):
 if schedule is not None:
 self.schedule = schedule
+if identifier is not None:
+self.identifier = identifier
 
 assert self.identifier is not None, "self.identifier must be set"
 assert self.schedule is not None, "self.schedule must be set"
@@ -120,23 +122,23 @@ class DeleteOldReports(ScheduledTask):
 measurement_path.child(measurement['id']).remove()
 
 
-class RunDecks(ScheduledTask):
+class RunDeck(ScheduledTask):
 """
 This will run the decks that have been configured on the system as the
 decks to run by default.
 """
-schedule = '@daily'
-identifier = 'run-decks'
 
-def __init__(self, director, schedule=None):
-super(RunDecks, self).__init__(schedule)
+def __init__(self, director, deck_id, schedule):
+self.deck_id = deck_id
 self.director = director
+identifier = 'run-deck-' + deck_id
+super(RunDeck, self).__init__(schedule, identifier)
 
 @defer.inlineCallbacks
 def task(self):
-for deck_id, deck in deck_store.list_enabled():
-yield deck.setup()
-yield deck.run(self.director)
+deck = deck_store.get(self.deck_id)
+yield deck.setup()
+yield deck.run(self.director)
 
 class SendHeartBeat(ScheduledTask):
 """
@@ -215,7 +217,10 @@ class SchedulerService(service.MultiService):
 self.schedule(UpdateInputsAndResources())
 self.schedule(UploadReports())
 self.schedule(DeleteOldReports())
-self.schedule(RunDecks(self.director))
+for deck_id, deck in deck_store.list_enabled():
+if deck.schedule is None:
+continue
+self.schedule(RunDeck(self.director, deck_id, deck.schedule))
 
 self._looping_call.start(self.interval)
 
diff --git a/ooni/scripts/ooniprobe.py b/ooni/scripts/ooniprobe.py
index f5d5b59..430252a 100644
--- a/ooni/scripts/ooniprobe.py
+++ b/ooni/scripts/ooniprobe.py
@@ -6,12 +6,14 @@ from twisted.internet import task, defer
 
 def ooniprobe(reactor):
 from ooni.ui.cli import runWithDaemonDirector, runWithDirector
-from ooni.ui.cli import setupGlobalOptions
+from ooni.ui.cli import setupGlobalOptions, initializeOoniprobe
 
 global_options = setupGlobalOptions(logging=True, start_tor=True,
 check_incoherences=True)
 if global_options['queue']:
 return runWithDaemonDirector(global_options)
+elif global_options['initialize']:
+return initializeOoniprobe(global_options)
 elif global_options['web-ui']:
 from ooni.scripts.ooniprobe_agent import WEB_UI_URL
 from ooni.scripts.ooniprobe_agent import status_agent, start_agent
diff --git a/ooni/settings.py b/ooni/settings.py
index 2161560..8bb3340 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -13,6 +13,125 @@ from ooni.utils.net import ConnectAndCloseProtocol, 
connectProtocol
 from ooni.utils import Storage, log, get_ooni_root
 from ooni import errors
 
+
+CONFIG_FILE_TEMPLATE = """\
+# This is the configuration file for OONIProbe
+# This file follows the YAML markup format: http://yaml.org/spec/1.2/spec.html
+# Keep in mind that indentation matters.
+
+basic:
+# Where OONIProbe should be writing it's log file
+logfile: {logfile}
+loglevel: WARNING
+privacy:
+# Should we include the IP address of the probe in the report?
+includeip: {include_ip}
+# Should we include the ASN of the probe in the report?
+includeasn: {include_asn}
+# Should we 

[tor-commits] [ooni-probe/master] Merge branch 'master' into v2.0.0-alpha

[art]

commit 847a951b40d1355c7f18d8e91001ad9a1e5fc984
Merge: 8048dbd da00195
Author: Arturo Filastò 
Date:   Fri Aug 26 18:33:38 2016 +0200

Merge branch 'master' into v2.0.0-alpha

* master:
  Don't look for the namebench csv file since we don't ship it




___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Use errno instead of integer to detect if file already exists

[art]

commit 1952ca0f1d55d8b78a1bed22c150bdfaf288b94e
Author: Arturo Filastò 
Date:   Thu Sep 1 17:30:45 2016 +0200

Use errno instead of integer to detect if file already exists
---
 ooni/settings.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ooni/settings.py b/ooni/settings.py
index 065252b..a0ef0a5 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -1,6 +1,7 @@
 import os
 import sys
 import yaml
+import errno
 import getpass
 from ConfigParser import SafeConfigParser
 
@@ -332,7 +333,7 @@ class OConfig(object):
 try:
 os.makedirs(path)
 except OSError as exc:
-if exc.errno != 17:
+if exc.errno != errno.EEXIST:
 raise
 
 def create_config_file(self, include_ip=False, include_asn=True,



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Improvements to start, stop, status workflow of ooniprobe-agent

[art]

commit e992f35ca6175ec53ec9966ca17966d86c129e8b
Author: Arturo Filastò 
Date:   Thu Sep 1 18:34:48 2016 +0200

Improvements to start, stop, status workflow of ooniprobe-agent

* Better support for running it across different users
---
 ooni/scripts/ooniprobe_agent.py | 64 +
 1 file changed, 46 insertions(+), 18 deletions(-)

diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py
index ecfe7d8..f56f2df 100644
--- a/ooni/scripts/ooniprobe_agent.py
+++ b/ooni/scripts/ooniprobe_agent.py
@@ -2,6 +2,7 @@ from __future__ import print_function
 
 import os
 import time
+import errno
 import signal
 
 from twisted.scripts import twistd
@@ -55,7 +56,8 @@ def start_agent(options=None):
 
 # Since we are starting the logger below ourselves we make twistd log to
 #  a null log observer
-twistd_args = ['--logger', 'ooni.utils.log.ooniloggerNull']
+twistd_args = ['--logger', 'ooni.utils.log.ooniloggerNull',
+   '--umask', '022']
 twistd_config = OoniprobeTwistdConfig()
 if options is not None:
 twistd_args.extend(options.twistd_args)
@@ -68,9 +70,12 @@ def start_agent(options=None):
 "StartOoniprobeAgent": StartOoniprobeAgentPlugin()
 }
 
-if status_agent() == 0:
+try:
+get_running_pidfile()
 print("Stop ooniprobe-agent before attempting to start it")
 return 1
+except NotRunning:
+pass
 
 print("Starting ooniprobe agent.")
 WEB_UI_URL = "http://{0}:{1}".format(
@@ -80,8 +85,16 @@ def start_agent(options=None):
 twistd.runApp(twistd_config)
 return 0
 
-def status_agent():
-running = False
+
+class NotRunning(RuntimeError):
+pass
+
+def get_running_pidfile():
+"""
+:return: This pid of the running ooniprobe-agent instance.
+:raises: NotRunning if it's not running
+"""
+running_pidfile = None
 for pidfile in [config.system_pid_path, config.user_pid_path]:
 if not os.path.exists(pidfile):
 # Didn't find the pid_file
@@ -90,36 +103,51 @@ def status_agent():
 pid = int(pid)
 try:
 os.kill(pid, signal.SIG_DFL)
-running = True
-except OSError, oserr:
-if oserr.errno == 3:
+running_pidfile = pidfile
+break
+except OSError as ose:
+if ose.errno == errno.ESRCH:
 # Found pid, but isn't running
 continue
-if running is True:
+elif ose.errno == errno.EPERM:
+# The process is owned by root. We assume it's running
+running_pidfile = pidfile
+break
+if running_pidfile is None:
+raise NotRunning
+return running_pidfile
+
+def status_agent():
+try:
+get_running_pidfile()
 print("ooniprobe-agent is running")
 return 0
-print("ooniprobe-agent is NOT running")
-return 1
+except NotRunning:
+print("ooniprobe-agent is NOT running")
+return 1
 
 def stop_agent():
 # This function is borrowed from tahoe
-pidfile = os.path.join(
-config.running_path,
-'twistd.pid'
-)
-if not os.path.exists(pidfile):
-print("It seems like ooniprobe-agent is not running")
+try:
+pidfile = get_running_pidfile()
+except NotRunning:
+print("ooniprobe-agent is NOT running. Nothing to do.")
 return 2
+
 pid = open(pidfile, "r").read()
 pid = int(pid)
 try:
 os.kill(pid, signal.SIGKILL)
-except OSError, oserr:
-if oserr.errno == 3:
+except OSError as ose:
+if ose.errno == errno.ESRCH:
 print("No process was running. Cleaning up.")
 # the process didn't exist, so wipe the pid file
 os.remove(pidfile)
 return 2
+elif ose.errno == errno.EPERM:
+# The process is owned by root. We assume it's running
+print("ooniprobe-agent is owned by root. We cannot stop it.")
+return 3
 else:
 raise
 try:



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Fix unittests for director, reporter and oonicli

[art]

commit 539c491e304a89ef2e57cb49aeeadbddab8d8baa
Author: Arturo Filastò 
Date:   Fri Aug 5 17:45:47 2016 +0200

Fix unittests for director, reporter and oonicli
---
 ooni/director.py|  1 -
 ooni/templates/httpt.py |  4 ++--
 ooni/tests/bases.py |  1 +
 ooni/tests/test_director.py |  5 +++--
 ooni/tests/test_oonicli.py  | 10 +++---
 ooni/tests/test_reporter.py |  4 +++-
 6 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/ooni/director.py b/ooni/director.py
index 304a14a..ab31bf5 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -373,7 +373,6 @@ class Director(object):
 Launches a Tor with :param: socks_port :param: control_port
 :param: tor_binary set in ooniprobe.conf
 """
-from txtorcon import TorConfig
 if self._tor_state == 'running':
 log.debug("Tor is already running")
 defer.returnValue(self._tor_state)
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index cba8702..d1aa1dc 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -110,8 +110,8 @@ class HTTPTest(NetTestCase):
 self.agent = FixedRedirectAgent(self.agent)
 self.report['agent'] = 'redirect'
 except:
-log.err("Warning! You are running an old version of twisted"\
-"(<= 10.1). I will not be able to follow redirects."\
+log.err("Warning! You are running an old version of twisted "
+"(<= 10.1). I will not be able to follow redirects."
 "This may make the testing less precise.")
 
 if len(self.contentDecoders) > 0:
diff --git a/ooni/tests/bases.py b/ooni/tests/bases.py
index 40e3b5e..5a7da5f 100644
--- a/ooni/tests/bases.py
+++ b/ooni/tests/bases.py
@@ -11,6 +11,7 @@ class ConfigTestCase(unittest.TestCase):
 self.ooni_home_dir = os.path.abspath("ooni_home")
 self.config = config
 self.config.initialize_ooni_home(self.ooni_home_dir)
+config.is_initialized = lambda: True
 super(ConfigTestCase, self).setUp()
 
 def skipTest(self, reason):
diff --git a/ooni/tests/test_director.py b/ooni/tests/test_director.py
index 6638adb..e5ca084 100644
--- a/ooni/tests/test_director.py
+++ b/ooni/tests/test_director.py
@@ -72,10 +72,11 @@ class TestDirector(ConfigTestCase):
 def test_start_tor(self):
 @defer.inlineCallbacks
 def director_start_tor():
+self.config.advanced.start_tor = True
 director = Director()
 yield director.start_tor()
-assert config.tor.socks_port == 4242
-assert config.tor.control_port == 4242
+self.assertEqual(config.tor.socks_port, 4242)
+self.assertEqual(config.tor.control_port, 4242)
 
 return director_start_tor()
 
diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py
index 06a21ff..8ad8a36 100644
--- a/ooni/tests/test_oonicli.py
+++ b/ooni/tests/test_oonicli.py
@@ -66,8 +66,6 @@ class TestRunDirector(ConfigTestCase):
 if not is_internet_connected():
 self.skipTest("You must be connected to the internet to run this 
test")
 
-config.tor.socks_port = 9050
-config.tor.control_port = None
 self.filenames = ['example-input.txt']
 with open('example-input.txt', 'w+') as f:
 f.write('http://torproject.org/\n')
@@ -83,7 +81,7 @@ class TestRunDirector(ConfigTestCase):
 
 @defer.inlineCallbacks
 def run_helper(self, test_name, nettest_args, verify_function, 
ooni_args=()):
-output_file = os.path.abspath('test_report.yamloo')
+output_file = os.path.abspath('test_report.njson')
 self.filenames.append(output_file)
 oldargv = sys.argv
 sys.argv = ['']
@@ -91,6 +89,12 @@ class TestRunDirector(ConfigTestCase):
 sys.argv.extend(['-n', '-o', output_file, test_name])
 sys.argv.extend(nettest_args)
 global_options = setupGlobalOptions(False, False, False)
+
+config.tor.socks_port = 9050
+config.advanced.start_tor = False
+config.tor.control_port = None
+config.advanced.debug = True
+
 yield runWithDirector(global_options,
   create_input_store=False)
 with open(output_file) as f:
diff --git a/ooni/tests/test_reporter.py b/ooni/tests/test_reporter.py
index cbfdaeb..f7a917d 100644
--- a/ooni/tests/test_reporter.py
+++ b/ooni/tests/test_reporter.py
@@ -3,6 +3,7 @@ import yaml
 import json
 import time
 import shutil
+import tempfile
 
 from twisted.internet import defer
 from twisted.trial import unittest
@@ -119,8 +120,8 @@ class TestOONIBReportLog(ConfigTestCase):
 
 def setUp(self):
 super(TestOONIBReportLog, self).setUp()
-self.report_log = OONIBReportLog()
 self.measurement_id = '20160727T182604Z-ZZ-AS0-dummy'
+

[tor-commits] [ooni-probe/master] Implement disk quota management

[art]

commit def4c929d162852ace8a016bb3352677eec5bcde
Author: Arturo Filastò 
Date:   Tue Aug 30 02:09:55 2016 +0200

Implement disk quota management

This is related to the feature described in here: 
https://github.com/TheTorProject/lepidopter/issues/53
---
 ooni/agent/scheduler.py  | 67 
 ooni/measurements.py | 20 +++
 ooni/settings.py |  6 -
 ooni/tests/test_utils.py | 34 
 ooni/ui/web/server.py| 12 -
 ooni/utils/files.py  | 34 
 6 files changed, 166 insertions(+), 7 deletions(-)

diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 3389db1..54551f6 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -1,3 +1,6 @@
+import os
+import errno
+
 from datetime import datetime
 
 from twisted.application import service
@@ -8,6 +11,7 @@ from twisted.python.filepath import FilePath
 from ooni.scripts import oonireport
 from ooni import resources
 from ooni.utils import log, SHORT_DATE
+from ooni.utils.files import human_size_to_bytes, directory_usage
 from ooni.deck.store import input_store, deck_store
 from ooni.settings import config
 from ooni.contrib import croniter
@@ -146,6 +150,66 @@ class DeleteOldReports(ScheduledTask):
 measurement_path.child(measurement['id']).remove()
 
 
+class CheckMeasurementQuota(ScheduledTask):
+"""
+This task is run to ensure we don't run out of disk space and deletes
+older reports to avoid filling the quota.
+"""
+identifier = 'check-measurement-quota'
+schedule = '@hourly'
+_warn_when = 0.8
+
+def task(self):
+if config.basic.measurement_quota is None:
+return
+maximum_bytes = human_size_to_bytes(config.basic.measurement_quota)
+available_bytes = directory_usage(config.measurements_directory)
+warning_path = os.path.join(config.running_path, 'quota_warning')
+
+if (float(available_bytes) / float(maximum_bytes)) >= self._warn_when:
+log.warn("You are about to reach the maximum allowed quota. Be 
careful")
+with open(warning_path, "w") as out_file:
+out_file.write("{0} {1}".split(available_bytes, maximum_bytes))
+else:
+try:
+os.remove(warning_path)
+except OSError as ose:
+if ose.errno != errno.ENOENT:
+raise
+
+if float(available_bytes) < float(maximum_bytes):
+# We are within the allow quota exit.
+return
+
+# We should begin to delete old reports
+amount_to_delete = float(maximum_bytes) - float(available_bytes)
+amount_deleted = 0
+measurement_path = FilePath(config.measurements_directory)
+
+kept_measurements = []
+stale_measurements = []
+remaining_measurements = []
+measurements_by_date = sorted(list_measurements(compute_size=True),
+  key=lambda k: k['test_start_time'])
+for measurement in measurements_by_date:
+if measurement['keep'] is True:
+kept_measurements.append(measurement)
+elif measurement['stale'] is True:
+stale_measurements.append(measurement)
+else:
+remaining_measurements.append(measurement)
+
+# This is the order in which we should begin deleting measurements.
+ordered_measurements = (stale_measurements +
+remaining_measurements +
+kept_measurements)
+while amount_deleted < amount_to_delete:
+measurement = ordered_measurements.pop(0)
+log.warn("Deleting report {0}".format(measurement["id"]))
+measurement_path.child(measurement['id']).remove()
+amount_deleted += measurement['size']
+
+
 class RunDeck(ScheduledTask):
 """
 This will run the decks that have been configured on the system as the
@@ -196,6 +260,7 @@ SYSTEM_TASKS = [
 UpdateInputsAndResources
 ]
 
+
 @defer.inlineCallbacks
 def run_system_tasks(no_input_store=False):
 task_classes = SYSTEM_TASKS[:]
@@ -215,6 +280,7 @@ def run_system_tasks(no_input_store=False):
 log.err("Failed to run task {0}".format(task.identifier))
 log.exception(exc)
 
+
 class SchedulerService(service.MultiService):
 """
 This service is responsible for running the periodic tasks.
@@ -271,6 +337,7 @@ class SchedulerService(service.MultiService):
 self.schedule(UpdateInputsAndResources())
 self.schedule(UploadReports())
 self.schedule(DeleteOldReports())
+self.schedule(CheckMeasurementQuota())
 self.schedule(RefreshDeckList(self))
 
 self._looping_call.start(self.interval)
diff --git a/ooni/measurements.py b/ooni/measurements.py
index 6d90c1b..fd722ee 

[tor-commits] [ooni-probe/master] Add to setup.py support for installing the updater

[art]

commit 21cf4b8525e911e89ee5c567ae5d375f88fda767
Author: Arturo Filastò 
Date:   Mon Sep 5 16:36:13 2016 +0200

Add to setup.py support for installing the updater
---
 MANIFEST.in |   5 +-
 data/configs/lepidopter-ooniprobe.conf  |  75 ---
 data/configs/lepidopter-oonireport.conf |  69 ---
 data/updater.py | 350 
 setup.py|  26 ++-
 5 files changed, 375 insertions(+), 150 deletions(-)

diff --git a/MANIFEST.in b/MANIFEST.in
index 60d2ef9..0528d4b 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -6,7 +6,8 @@ include data/ooniprobe.1
 include data/oonireport.1
 include data/ooniresources.1
 include data/ooniprobe.conf.sample
-include data/configs/lepidopter-ooniprobe.conf
-include data/configs/lepidopter-oonireport.conf
+
+include data/updater.py
+
 include ooni/settings.ini
 include ooni/ui/consent-form.md
diff --git a/data/configs/lepidopter-ooniprobe.conf 
b/data/configs/lepidopter-ooniprobe.conf
deleted file mode 100644
index 5c8ba40..000
--- a/data/configs/lepidopter-ooniprobe.conf
+++ /dev/null
@@ -1,75 +0,0 @@
-# This is the configuration file for OONIProbe
-# This file follows the YAML markup format: http://yaml.org/spec/1.2/spec.html
-# Keep in mind that indentation matters.
-
-basic:
-# Where OONIProbe should be writing it's log file
-logfile: /var/log/ooni/ooniprobe.log
-# in the future we will support loglevels
-loglevel: WARNING
-privacy:
-# Should we include the IP address of the probe in the report?
-includeip: false
-# Should we include the ASN of the probe in the report?
-includeasn: true
-# Should we include the country as reported by GeoIP in the report?
-includecountry: true
-# Should we include the city as reported by GeoIP in the report?
-includecity: false
-# Should we collect a full packet capture on the client?
-includepcap: false
-reports:
-# Should we place a unique ID inside of every report
-unique_id: true
-# This is a prefix for each packet capture file (.pcap) per test:
-pcap: null
-collector: null
-advanced:
-geoip_data_dir: /usr/share/GeoIP
-debug: false
-# enable if auto detection fails
-#tor_binary: /usr/sbin/tor
-#obfsproxy_binary: /usr/bin/obfsproxy 
-# For auto detection
-interface: auto
-# Of specify a specific interface
-#interface: wlan0
-# If you do not specify start_tor, you will have to have Tor running and
-# explicitly set the control port and SOCKS port
-start_tor: true
-# After how many seconds we should give up on a particular measurement
-measurement_timeout: 120
-# After how many retries we should give up on a measurement
-measurement_retries: 2
-# How many measurements to perform concurrently
-measurement_concurrency: 4
-# After how may seconds we should give up reporting
-reporting_timeout: 360
-# After how many retries to give up on reporting
-reporting_retries: 5
-# How many reports to perform concurrently
-reporting_concurrency: 7
-oonid_api_port: 8042
-report_log_file: null
-inputs_dir: null
-decks_dir: null
-tor:
-#socks_port: 8801
-#control_port: 8802
-# Specify the absolute path to the Tor bridges to use for testing
-#bridges: bridges.list
-# Specify path of the tor datadirectory.
-# This should be set to something to avoid having Tor download each time
-# the descriptors and consensus data.
-data_dir: /opt/ooni/tor_data_dir
-torrc:
-#HTTPProxy: host:port
-#HTTPProxyAuthenticator: user:password
-#HTTPSProxy: host:port
-#HTTPSProxyAuthenticator: user:password
-# Uncomment following 5 lines to connect via meek pluggable transport in 
tor
-#UseBridges: 1
-#Bridge:
-#- "meek_lite 0.0.2.0:1 url=https://d2zfqthxsdq309.cloudfront.net/ 
front=a0.awsstatic.com"
-#- "meek_lite 0.0.2.0:2 url=https://az786092.vo.msecnd.net/ 
front=ajax.aspnetcdn.com"
-#ClientTransportPlugin: "meek_lite exec /usr/bin/obfs4proxy"
diff --git a/data/configs/lepidopter-oonireport.conf 
b/data/configs/lepidopter-oonireport.conf
deleted file mode 100644
index 8ea3ccc..000
--- a/data/configs/lepidopter-oonireport.conf
+++ /dev/null
@@ -1,69 +0,0 @@
-# This is the configuration file for OONIProbe
-# This file follows the YAML markup format: http://yaml.org/spec/1.2/spec.html
-# Keep in mind that indentation matters.
-
-basic:
-# Where OONIProbe should be writing it's log file
-logfile: /var/log/ooni/oonireport.log
-# in the future we will support loglevels
-loglevel: WARNING
-privacy:
-# Should we include the IP address of the probe in the report?
-includeip: false
-# Should we include the ASN of the probe in the report?
-includeasn: true
-# Should we include the country as reported by GeoIP in the report?
-includecountry: true
-# Should 

[tor-commits] [ooni-probe/master] Update the updater

[art]

commit 1046f0ec3692f57d80ee73488bcdabc56d6e315c
Author: Arturo Filastò 
Date:   Tue Sep 6 12:50:09 2016 +0200

Update the updater
---
 data/updater.py | 9 +
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/data/updater.py b/data/updater.py
index dc2fb2a..0f423ab 100755
--- a/data/updater.py
+++ b/data/updater.py
@@ -23,10 +23,10 @@ CURRENT_VERSION_PATH = "/etc/lepidopter-update/version"
 UPDATER_PATH = "/opt/ooni/updater/versions/"
 SCRIPT_INSTALL_PATH = "/opt/ooni/updater/updater.py"
 
-SYSTEMD_SCRIPT_PATH = "/etc/systemd/system/lepidopter-updater.service"
+SYSTEMD_SCRIPT_PATH = "/etc/systemd/system/lepidopter-update.service"
 SYSTEMD_SCRIPT = """\
 [Unit]
-Description=lepidopter-updater service
+Description=lepidopter-update service
 
 [Service]
 Type=simple
@@ -138,12 +138,13 @@ def verify_file(signature_path, signer_pk_path):
 
 try:
 try:
-check_call(["gpg", "--yes", "-o", tmp_key, "--dearmor", 
signer_pk_path])
+check_call(["gpg", "--batch", "--yes", "-o", tmp_key,
+"--dearmor", signer_pk_path])
 except CalledProcessError:
 raise InvalidPublicKey
 
 try:
-output = check_output(["gpg", "--status-fd", "1",
+output = check_output(["gpg", "--batch", "--status-fd", "1",
"--no-default-keyring", "--keyring",
tmp_key, "--trust-model", "always",
"--verify", signature_path])



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Fix typo in data deletion schedule

[art]

commit b41bc4b5c6e9616fd4a0ea50a770d862984379e2
Author: Arturo Filastò 
Date:   Mon Sep 5 15:54:40 2016 +0200

Fix typo in data deletion schedule
---
 ooni/agent/scheduler.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 7c2212b..3cbd29d 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -182,7 +182,8 @@ class CheckMeasurementQuota(ScheduledTask):
 if (float(available_bytes) / float(maximum_bytes)) >= self._warn_when:
 log.warn("You are about to reach the maximum allowed quota. Be 
careful")
 with open(warning_path, "w") as out_file:
-out_file.write("{0} {1}".split(available_bytes, maximum_bytes))
+out_file.write("{0} {1}".format(available_bytes,
+maximum_bytes))
 else:
 try:
 os.remove(warning_path)



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Make settings parsing more robust

[art]

commit b334679f58c835b7a93b6f93662b7cf7aab80240
Author: Arturo Filastò 
Date:   Fri Sep 2 17:35:36 2016 +0200

Make settings parsing more robust

* Expand test case to cover more edge cases
---
 ooni/settings.py| 10 +++---
 ooni/tests/test_settings.py | 12 +---
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/ooni/settings.py b/ooni/settings.py
index 66bd212..9f24f9b 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -89,7 +89,7 @@ tor:
 # This is the timeout after which we consider to to not have
 # bootstrapped properly.
 #timeout: 200
-torrc:
+#torrc:
 #HTTPProxy: host:port
 #HTTPProxyAuthenticator: user:password
 #HTTPSProxy: host:port
@@ -213,14 +213,18 @@ def _load_config_files_with_defaults(config_files, 
defaults):
 continue
 with open(config_file_path) as in_file:
 c = yaml.safe_load(in_file)
-config_from_files.update(c)
+for category in c.keys():
+if c[category] is None:
+continue
+config_from_files[category] = config_from_files.get(category, {})
+config_from_files[category].update(c[category])
 
 for category in defaults.keys():
 configuration[category] = {}
 for k, v in defaults[category].items():
 try:
 configuration[category][k] = config_from_files[category][k]
-except KeyError:
+except (KeyError, TypeError):
 configuration[category][k] = defaults[category][k]
 return configuration
 
diff --git a/ooni/tests/test_settings.py b/ooni/tests/test_settings.py
index 30d9e19..b88aa68 100644
--- a/ooni/tests/test_settings.py
+++ b/ooni/tests/test_settings.py
@@ -146,7 +146,8 @@ class TestSettings(ConfigTestCase):
 'key': 'value'
 },
 'cat2': {
-'key': 'value'
+'key': 'value',
+'key2': 'value2'
 },
 'cat3': {
 'key': 'value'
@@ -162,11 +163,15 @@ class TestSettings(ConfigTestCase):
 },
 'invalid_category': {
 'ignored': 'ignored'
-}
+},
+'cat3': None
 }
 config_file_B = {
 'cat1': {
 'key': 'valueB'
+},
+'cat2': {
+'key2': 'value2B'
 }
 }
 temp_dir = tempfile.mkdtemp()
@@ -188,7 +193,8 @@ class TestSettings(ConfigTestCase):
 'key': 'valueB'
 },
 'cat2': {
-'key': 'valueA'
+'key': 'valueA',
+'key2': 'value2B'
 },
 'cat3': {
 'key': 'value'



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Add ooniprobe systemd service

[art]

commit 8da2312d61c4c1a840d0878e19967629526cd731
Author: Arturo Filastò 
Date:   Tue Sep 6 11:24:31 2016 +0200

Add ooniprobe systemd service
---
 scripts/systemd/ooniprobe.service | 15 +++
 1 file changed, 15 insertions(+)

diff --git a/scripts/systemd/ooniprobe.service 
b/scripts/systemd/ooniprobe.service
new file mode 100644
index 000..4743fac
--- /dev/null
+++ b/scripts/systemd/ooniprobe.service
@@ -0,0 +1,15 @@
+[Unit]
+Description=%n, network interference detection tool
+After=network.target nss-lookup.target
+
+[Service]
+Type=forking
+PIDFile=/var/lib/ooni/twistd.pid
+ExecStart=/usr/local/bin/ooniprobe-agent start
+ExecStop=/usr/local/bin/ooniprobe-agent stop
+TimeoutStartSec=300
+TimeoutStopSec=60
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Bump version to rc.1

[art]

commit 7fe1bc1be0f657b8c2492408288067f2d4fd1d62
Author: Arturo Filastò 
Date:   Tue Sep 6 16:19:02 2016 +0200

Bump version to rc.1
---
 ooni/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ooni/__init__.py b/ooni/__init__.py
index d4b6f95..de2157c 100644
--- a/ooni/__init__.py
+++ b/ooni/__init__.py
@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 
 __author__ = "Open Observatory of Network Interference"
-__version__ = "2.0.0-alpha.3"
+__version__ = "2.0.0-rc.1"
 
 __all__ = [
 'agent',



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Move lepidopter update install process to the install script

[art]

commit 1165aeee44567e604f24291a716ef2fb6202e38f
Author: Arturo Filastò 
Date:   Mon Sep 5 16:48:56 2016 +0200

Move lepidopter update install process to the install script
---
 setup.py | 17 +++--
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/setup.py b/setup.py
index c465789..60d08a5 100644
--- a/setup.py
+++ b/setup.py
@@ -143,7 +143,7 @@ def is_updater_installed():
 return os.path.exists('/etc/lepidopter-update/version')
 
 
-def install_updater():
+def install_lepidopter_update():
 check_call(["data/updater.py", "install"])
 
 
@@ -199,17 +199,14 @@ class OoniInstall(InstallCommand):
 def run(self):
 self.pre_install()
 self.do_egg_install()
-
+if is_lepidopter() and not is_updater_installed():
+print("Lepidopter now requires that ooniprobe is installed via the 
"
+ "updater")
+print("Let me install the auto-updater for you and we shall use 
that "
+  "for updates in the future.")
+install_lepidopter_update()
 
 def setup_package():
-if is_lepidopter() and not is_updater_installed():
-print("Lepidopter now requires that ooniprobe is installed via the "
-  "updater")
-print("Let me install the auto-updater for you and we shall use that "
-  "for updates in the future.")
-install_updater()
-return
-
 setup_requires = []
 install_requires = []
 dependency_links = []



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Add support for parsing multiple config files

[art]

commit 31bcae623b8a621aa941f284e67438168243c44a
Author: Arturo Filastò 
Date:   Fri Sep 2 15:00:48 2016 +0200

Add support for parsing multiple config files

* Configuration files are parsed in a certain order and they override the
  default behavior.
---
 Vagrantfile |   5 +
 ooni/__init__.py|   2 +-
 ooni/scripts/ooniprobe_agent.py |   1 +
 ooni/settings.py| 125 ++-
 ooni/tests/test_settings.py |  61 ++
 ooni/ui/cli.py  |   4 +-
 ooni/ui/web/client/index.html   |   2 +-
 setup.py| 262 ++--
 8 files changed, 285 insertions(+), 177 deletions(-)

diff --git a/Vagrantfile b/Vagrantfile
index 71f7c7a..8ff0bbd 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -81,4 +81,9 @@ Vagrant.configure("2") do |config|
 end
   end
 
+  config.vm.define "testing" do |testing|
+testing.vm.network "forwarded_port", guest: 8842, host: 8142
+testing.vm.synced_folder ".", "/data/ooni-probe"
+  end
+
 end
diff --git a/ooni/__init__.py b/ooni/__init__.py
index 9eecb67..d4b6f95 100644
--- a/ooni/__init__.py
+++ b/ooni/__init__.py
@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 
 __author__ = "Open Observatory of Network Interference"
-__version__ = "2.0.0-alpha.2"
+__version__ = "2.0.0-alpha.3"
 
 __all__ = [
 'agent',
diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py
index f56f2df..3eb1d22 100644
--- a/ooni/scripts/ooniprobe_agent.py
+++ b/ooni/scripts/ooniprobe_agent.py
@@ -58,6 +58,7 @@ def start_agent(options=None):
 #  a null log observer
 twistd_args = ['--logger', 'ooni.utils.log.ooniloggerNull',
'--umask', '022']
+
 twistd_config = OoniprobeTwistdConfig()
 if options is not None:
 twistd_args.extend(options.twistd_args)
diff --git a/ooni/settings.py b/ooni/settings.py
index a0ef0a5..66bd212 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -3,6 +3,7 @@ import sys
 import yaml
 import errno
 import getpass
+from pkg_resources import parse_version
 from ConfigParser import SafeConfigParser
 
 from twisted.internet import defer, reactor
@@ -10,6 +11,7 @@ from twisted.internet.endpoints import TCP4ClientEndpoint
 
 from os.path import abspath, expanduser
 
+from ooni import __version__ as ooniprobe_version
 from ooni.utils import Storage, log, get_ooni_root
 
 CONFIG_FILE_TEMPLATE = """\
@@ -19,8 +21,8 @@ CONFIG_FILE_TEMPLATE = """\
 
 basic:
 # Where OONIProbe should be writing it's log file
-logfile: {logfile}
-loglevel: WARNING
+# logfile: {logfile}
+# loglevel: WARNING
 # The maximum amount of data to store on disk. Once the quota is reached,
 # we will start deleting older reports.
 # measurement_quota: 1G
@@ -191,6 +193,37 @@ elif os.path.isfile(_SETTINGS_INI):
 if _ETC_PATH is not None:
 ETC_PATH = _ETC_PATH
 
+
+def _load_config_files_with_defaults(config_files, defaults):
+"""
+This takes care of reading the config files in reverse order (the first
+item will have priority over the last element) and produce a
+configuration that includes ONLY the options inside of the defaults
+dictionary.
+
+:param config_files: a list of configuration file paths
+:param defaults: the default values for the configuration file
+:return: a configuration that is the result of reading the config files
+and joining it with the default options.
+"""
+config_from_files = {}
+configuration = {}
+for config_file_path in reversed(config_files):
+if not os.path.exists(config_file_path):
+continue
+with open(config_file_path) as in_file:
+c = yaml.safe_load(in_file)
+config_from_files.update(c)
+
+for category in defaults.keys():
+configuration[category] = {}
+for k, v in defaults[category].items():
+try:
+configuration[category][k] = config_from_files[category][k]
+except KeyError:
+configuration[category][k] = defaults[category][k]
+return configuration
+
 class OConfig(object):
 _custom_home = None
 
@@ -211,6 +244,10 @@ class OConfig(object):
 self.tor = Storage()
 self.privacy = Storage()
 
+# In here we store the configuration files ordered by priority.
+# First configuration file takes priority over the others.
+self.config_files = []
+
 self.set_paths()
 
 def is_initialized(self):
@@ -225,6 +262,31 @@ class OConfig(object):
 with open(initialized_path, 'w+'): pass
 
 @property
+def last_run_version(self):
+"""
+:return: Version identifying the last run version of ooniprobe.
+"""
+last_run_version_path = os.path.join(
+self.running_path, "last_run_version"
+)
+if not os.path.exists(last_run_version_path):
+  

[tor-commits] [ooni-probe/master] Bump to version a0

[art]

commit d1765a80349cc5eba46b342056cc08cc29ae6ae0
Author: Arturo Filastò 
Date:   Sat Jul 30 17:23:46 2016 +0200

Bump to version a0
---
 ooni/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ooni/__init__.py b/ooni/__init__.py
index c5c3c0a..653a636 100644
--- a/ooni/__init__.py
+++ b/ooni/__init__.py
@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 
 __author__ = "Open Observatory of Network Interference"
-__version__ = "2.0.0.dev1"
+__version__ = "2.0.0a0"
 
 __all__ = [
 'agent',



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Add support for deck lifecycle in the web UI

[art]

commit 7663106d25d272ffc7ff2208d95cddef3b685f62
Author: Arturo Filastò 
Date:   Thu Aug 4 19:46:26 2016 +0200

Add support for deck lifecycle in the web UI

* Fix a series of bugs found while testing
* Bump the version number up one
---
 MANIFEST.in|   1 +
 data/decks/web-full.yaml   |  23 ++
 data/decks/web-no-invalid.yaml |  18 +
 data/decks/web.yaml|  23 --
 ooni/__init__.py   |   2 +-
 ooni/agent/agent.py|   9 ++-
 ooni/agent/scheduler.py|  64 ++---
 ooni/deck/deck.py  |   9 ++-
 ooni/deck/store.py |   4 +-
 ooni/resources.py  |   5 ++
 ooni/scripts/oonideckgen.py|  20 ++
 ooni/scripts/ooniprobe.py  |   3 +-
 ooni/scripts/oonireport.py |   7 +-
 ooni/settings.py   | 154 +++--
 ooni/ui/cli.py |  37 --
 ooni/ui/web/client/index.html  |   2 +-
 ooni/ui/web/server.py  |  28 +++-
 ooni/ui/web/web.py |   5 +-
 ooni/utils/__init__.py |   4 +-
 setup.py   |  29 +++-
 20 files changed, 271 insertions(+), 176 deletions(-)

diff --git a/MANIFEST.in b/MANIFEST.in
index 485e834..60d2ef9 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -9,3 +9,4 @@ include data/ooniprobe.conf.sample
 include data/configs/lepidopter-ooniprobe.conf
 include data/configs/lepidopter-oonireport.conf
 include ooni/settings.ini
+include ooni/ui/consent-form.md
diff --git a/data/decks/web-full.yaml b/data/decks/web-full.yaml
new file mode 100644
index 000..7812505
--- /dev/null
+++ b/data/decks/web-full.yaml
@@ -0,0 +1,23 @@
+---
+name: Full Web test deck
+description: This deck runs HTTP Header Field Manipulation, HTTP Invalid
+Request and the Web Connectivity test
+schedule: "@daily"
+tasks:
+- name: Runs the HTTP Header Field Manipulation test
+  ooni:
+test_name: http_header_field_manipulation
+
+- name: Runs the HTTP Invalid Request Line test
+  ooni:
+test_name: http_invalid_request_line
+
+- name: Runs the Web Connectivity Test
+  ooni:
+test_name: web_connectivity
+file: $citizenlab_global_urls
+
+- name: Runs the Web Connectivity Test
+  ooni:
+test_name: web_connectivity
+file: $citizenlab_${probe_cc}_urls
diff --git a/data/decks/web-no-invalid.yaml b/data/decks/web-no-invalid.yaml
new file mode 100644
index 000..ea93488
--- /dev/null
+++ b/data/decks/web-no-invalid.yaml
@@ -0,0 +1,18 @@
+---
+name: Web test deck without HTTP Invalid Request Line
+description: This deck runs HTTP Header Field Manipulation, and the Web 
Connectivity test
+schedule: "@daily"
+tasks:
+- name: Runs the HTTP Header Field Manipulation test
+  ooni:
+test_name: http_header_field_manipulation
+
+- name: Runs the Web Connectivity Test
+  ooni:
+test_name: web_connectivity
+file: $citizenlab_global_urls
+
+- name: Runs the Web Connectivity Test
+  ooni:
+test_name: web_connectivity
+file: $citizenlab_${probe_cc}_urls
diff --git a/data/decks/web.yaml b/data/decks/web.yaml
deleted file mode 100644
index c7b9bdc..000
--- a/data/decks/web.yaml
+++ /dev/null
@@ -1,23 +0,0 @@

-name: Web related ooniprobe tests
-description: This deck runs HTTP Header Field Manipulation, HTTP Invalid
-Request and the Web Connectivity test
-schedule: "@daily"
-tasks:
-- name: Runs the HTTP Header Field Manipulation test
-  ooni:
-test_name: http_header_field_manipulation
-
-- name: Runs the HTTP Invalid Request Line test
-  ooni:
-test_name: http_invalid_request_line
-
-- name: Runs the Web Connectivity Test
-  ooni:
-test_name: web_connectivity
-file: $citizenlab_global_urls
-
-- name: Runs the Web Connectivity Test
-  ooni:
-test_name: web_connectivity
-file: $citizenlab_${probe_cc}_urls
diff --git a/ooni/__init__.py b/ooni/__init__.py
index 653a636..1a31608 100644
--- a/ooni/__init__.py
+++ b/ooni/__init__.py
@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 
 __author__ = "Open Observatory of Network Interference"
-__version__ = "2.0.0a0"
+__version__ = "2.0.0a1"
 
 __all__ = [
 'agent',
diff --git a/ooni/agent/agent.py b/ooni/agent/agent.py
index c2e7e26..0311cef 100644
--- a/ooni/agent/agent.py
+++ b/ooni/agent/agent.py
@@ -11,12 +11,15 @@ class AgentService(service.MultiService):
 
 director = Director()
 
-self.web_ui_service = WebUIService(director, web_ui_port)
-self.web_ui_service.setServiceParent(self)
-
 self.scheduler_service = SchedulerService(director)
 self.scheduler_service.setServiceParent(self)
 
+self.web_ui_service = WebUIService(director,
+   self.scheduler_service,
+   web_ui_port)
+self.web_ui_service.setServiceParent(self)
+
+
 def startService(self):
 service.MultiService.startService(self)
 
diff --git 

[tor-commits] [ooni-probe/master] Ignore global options that are None or set to zero.

[art]

commit 196fc17c2d62a8eb37e95e55f0dedaa2c53284f5
Author: Arturo Filastò 
Date:   Mon Aug 29 09:39:14 2016 +0200

Ignore global options that are None or set to zero.

* Always pop values from task_data to ensure the options are always clean.

This closes #592
---
 ooni/deck/deck.py   | 16 ++--
 ooni/tests/test_deck.py | 10 +-
 ooni/ui/cli.py  |  4 ++--
 3 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/ooni/deck/deck.py b/ooni/deck/deck.py
index 037c5d6..e4c35c4 100644
--- a/ooni/deck/deck.py
+++ b/ooni/deck/deck.py
@@ -317,12 +317,16 @@ class DeckTask(object):
 
 self._load(data)
 
-def _get_option(self, name, task_data, default=None):
+def _pop_option(self, name, task_data, default=None):
 try:
-return self.global_options[name]
+value = self.global_options[name]
+if value in [None, 0]:
+raise KeyError
 except KeyError:
-return task_data.pop(name,
- self.parent_metadata.get(name, default))
+value = task_data.pop(name,
+  self.parent_metadata.get(name, default))
+task_data.pop(name, None)
+return value
 
 def _load_ooni(self, task_data):
 required_keys = ["test_name"]
@@ -334,8 +338,8 @@ class DeckTask(object):
 nettest_path = nettest_to_path(task_data.pop("test_name"),
self._arbitrary_paths)
 
-annotations = self._get_option('annotations', task_data, {})
-collector_address = self._get_option('collector', task_data, None)
+annotations = self._pop_option('annotations', task_data, {})
+collector_address = self._pop_option('collector', task_data, None)
 
 try:
 self.output_path = self.global_options['reportfile']
diff --git a/ooni/tests/test_deck.py b/ooni/tests/test_deck.py
index 1bc6097..b2d05a6 100644
--- a/ooni/tests/test_deck.py
+++ b/ooni/tests/test_deck.py
@@ -14,7 +14,7 @@ from hashlib import sha256
 from ooni import errors
 from ooni.deck.store import input_store
 from ooni.deck.backend import lookup_collector_and_test_helpers
-from ooni.deck.deck import nettest_to_path, NGDeck
+from ooni.deck.deck import nettest_to_path, NGDeck, options_to_args
 from ooni.deck.legacy import convert_legacy_deck
 from ooni.tests.bases import ConfigTestCase
 from ooni.tests.mocks import MockBouncerClient, MockCollectorClient
@@ -330,3 +330,11 @@ class TestNGDeck(ConfigTestCase):
 "manipulation/http_header_field_manipulation",
 "blocking/web_connectivity"
 ])
+tasks = map(lambda task: task['ooni'], ng_deck['tasks'])
+self.assertEqual(
+tasks[2]['f'],
+'/path/to/citizenlab-urls-global.txt')
+
+def test_options_to_args(self):
+args = options_to_args({"f": "foobar.txt", "bar": None, "help": 0})
+print(args)
diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py
index 61f254b..3ad9605 100644
--- a/ooni/ui/cli.py
+++ b/ooni/ui/cli.py
@@ -338,8 +338,8 @@ def createDeck(global_options, url=None):
 raise SystemExit(3)
 
 except errors.OONIUsageError as e:
-log.err(e)
-print e.net_test_loader.usageOptions().getUsage()
+log.exception(e)
+map(log.msg, e.net_test_loader.usageOptions().getUsage().split("\n"))
 raise SystemExit(4)
 
 except errors.HTTPSCollectorUnsupported:



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Don't look for the namebench csv file since we don't ship it

[art]

commit da00195094cdf22678d42608cdbc3bb22b9d757e
Author: Arturo Filastò 
Date:   Wed Aug 10 18:11:06 2016 +0200

Don't look for the namebench csv file since we don't ship it

* Include an empty file so that older ooniprobes don't end up always 
updating
  the resources on every cycle.
---
 ooni/deckgen/cli.py | 4 
 setup.py| 3 +++
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/ooni/deckgen/cli.py b/ooni/deckgen/cli.py
index a6046f3..27e465a 100644
--- a/ooni/deckgen/cli.py
+++ b/ooni/deckgen/cli.py
@@ -122,10 +122,6 @@ def resources_up_to_date():
 return False
 
 if config.get_data_file_path("resources/"
- "namebench-dns-servers.csv") is None:
-return False
-
-if config.get_data_file_path("resources/"
  "citizenlab-test-lists/"
  "global.csv") is None:
 return False
diff --git a/setup.py b/setup.py
index ff1b6c2..b175662 100644
--- a/setup.py
+++ b/setup.py
@@ -279,6 +279,9 @@ class CreateOoniResources(Command):
 pj(pkg_dir, "GeoIP", "GeoIPASNum.dat"))
 shutil.move(pj(tmp_dir, "test-lists-master", "lists"),
 pj(pkg_dir, "resources", "citizenlab-test-lists"))
+# Touch the namebench dns servers file
+with open(pj(pkg_dir, "resources", "namebench-dns-servers.csv"), "w"):
+pass
 # Don't include services and official lists
 shutil.rmtree(
 pj(pkg_dir,



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Merge pull request #581 from TheTorProject/feature/webui2-rb

[art]

commit 0b1ffa329ee7db8e11131f59f2113c9e55de3ee4
Merge: eaf15e6 f3d3bb1
Author: Arturo Filastò 
Date:   Mon Aug 8 16:43:59 2016 +0200

Merge pull request #581 from TheTorProject/feature/webui2-rb

Feature/webui2 rb

 .gitignore |1 +
 MANIFEST.in|3 +
 Vagrantfile|   33 +-
 bin/Makefile   |   81 --
 bin/oonid  |7 -
 bin/oonideckgen|   37 -
 bin/ooniprobe  |   24 -
 bin/ooniprobe-dev  |   10 -
 bin/oonireport |   38 -
 bin/ooniresources  |   35 -
 bin/test/ooni/__init__.py  |1 -
 data/component-template/templ.component.js |   10 -
 data/component-template/templ.controller.js|5 -
 data/component-template/templ.css  |0
 data/component-template/templ.html |8 -
 data/component-template/templ.js   |   19 -
 data/decks/web-full.yaml   |   23 +
 data/decks/web-no-invalid.yaml |   18 +
 data/ooniprobe.conf.sample |5 +-
 ooni/__init__.py   |   13 +-
 ooni/agent/__init__.py |0
 ooni/agent/agent.py|   27 +
 ooni/agent/scheduler.py|  280 
 ooni/backend_client.py |  117 +-
 ooni/constants.py  |4 +-
 ooni/contrib/__init__.py   |1 +
 ooni/contrib/croniter.py   |  430 +++
 ooni/contrib/dateutil/__init__.py  |0
 ooni/contrib/dateutil/relativedelta.py |  539 
 ooni/contrib/dateutil/tz/__init__.py   |4 +
 ooni/contrib/dateutil/tz/_common.py|  100 ++
 ooni/contrib/dateutil/tz/tz.py | 1339 
 ooni/contrib/dateutil/tz/win.py|  354 ++
 ooni/deck.py   |  413 --
 ooni/deck/__init__.py  |1 +
 ooni/deck/backend.py   |  191 +++
 ooni/deck/deck.py  |  405 ++
 ooni/deck/legacy.py|   65 +
 ooni/deck/store.py |  192 +++
 ooni/deckgen/__init__.py   |1 -
 ooni/deckgen/cli.py|  183 ---
 ooni/deckgen/processors/__init__.py|0
 ooni/deckgen/processors/citizenlab_test_lists.py   |   55 -
 ooni/deckgen/processors/namebench_dns_servers.py   |   51 -
 ooni/director.py   |  227 ++--
 ooni/geoip.py  |   57 +-
 ooni/measurements.py   |  151 ++-
 ooni/nettest.py|  118 +-
 ooni/nettests/blocking/web_connectivity.py |   14 +-
 .../manipulation/http_invalid_request_line.py  |   14 +-
 ooni/oonicli.py|  520 
 ooni/otime.py  |7 +
 ooni/report/__init__.py|1 -
 ooni/report/cli.py |   90 --
 ooni/report/parser.py  |   35 -
 ooni/report/tool.py|  117 --
 ooni/reporter.py   |  428 ---
 ooni/resources.py  |  184 +++
 ooni/resources/__init__.py |7 -
 ooni/resources/cli.py  |   44 -
 ooni/resources/update.py   |   57 -
 ooni/scripts/__init__.py   |0
 ooni/scripts/oonideckgen.py|  138 ++
 ooni/scripts/ooniprobe.py  |   35 +
 ooni/scripts/ooniprobe_agent.py|  165 +++
 ooni/scripts/oonireport.py |  309 +
 ooni/scripts/ooniresources.py  |   34 +
 ooni/settings.py   |  364 --
 ooni/templates/httpt.py|9 +-
 ooni/templates/process.py  |7 +-
 ooni/templates/tcpt.py |1 -
 ooni/tests/__init__.py |7 -
 ooni/tests/bases.py|4 +-
 ooni/tests/mocks.py|2 +-
 ooni/tests/test_backend_client.py  |   16 -
 ooni/tests/test_deck.py|  271 ++--
 

[tor-commits] [ooni-probe/master] Run the check_for_update function when installation is complete

[art]

commit f7fec4b6d0aca457cfafdb0617218f2b4c0f0345
Author: Arturo Filastò 
Date:   Sat Jul 30 17:53:32 2016 +0200

Run the check_for_update function when installation is complete
---
 Vagrantfile |   2 +-
 ooni/agent/scheduler.py |   6 ++-
 setup.py| 114 +++-
 3 files changed, 10 insertions(+), 112 deletions(-)

diff --git a/Vagrantfile b/Vagrantfile
index cf73244..0e75d55 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -13,7 +13,7 @@ apt-get update
 apt-get install -y tor deb.torproject.org-keyring
 
 # Setup for sniffer subsystem
-apt-get install -y build-essential libdumbnet-dev libpcap-dev libgeoip-dev 
libffi-dev python-dev python-pip
+apt-get install -y build-essential libdumbnet-dev libpcap-dev libgeoip-dev 
libffi-dev python-dev python-pip libssl-dev
 cd /data/ooni-probe
 python setup.py install
 
diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 7a77afb..0d8689c 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -1,7 +1,8 @@
 from datetime import datetime
 
 from twisted.application import service
-from twisted.internet import task, defer
+from twisted.internet import defer
+from twisted.internet.task import LoopingCall
 from twisted.python.filepath import FilePath
 
 from ooni.scripts import oonireport
@@ -72,6 +73,7 @@ class ScheduledTask(object):
 finally:
 self._last_run_lock.unlock()
 
+
 class UpdateInputsAndResources(ScheduledTask):
 identifier = "update-inputs"
 schedule = "@daily"
@@ -177,7 +179,7 @@ class SchedulerService(service.MultiService):
 service.MultiService.__init__(self)
 self.director = director
 self.interval = interval
-self._looping_call = task.LoopingCall(self._should_run)
+self._looping_call = LoopingCall(self._should_run)
 self._scheduled_tasks = []
 
 def schedule(self, task):
diff --git a/setup.py b/setup.py
index 6940f66..da48323 100644
--- a/setup.py
+++ b/setup.py
@@ -89,16 +89,14 @@ Have fun!
 from __future__ import print_function
 
 import os
-import sys
 import shutil
 import tempfile
 import subprocess
 from ConfigParser import SafeConfigParser
 
 from os.path import join as pj
-from setuptools import setup, Command
+from setuptools import setup
 from setuptools.command.install import install
-from distutils.spawn import find_executable
 
 from ooni import __version__, __author__
 
@@ -177,11 +175,9 @@ class OoniInstall(install):
 pass
 
 def ooniresources(self):
-ooniresources = find_executable("ooniresources")
-process = subprocess.Popen([ooniresources],
-   stdout=sys.stdout.fileno(),
-   stderr=sys.stderr.fileno())
-process.wait()
+from ooni.resources import check_for_update
+from twisted.internet import task
+task.react(lambda _: check_for_update())
 
 def update_lepidopter_config(self):
 try:
@@ -200,105 +196,6 @@ class OoniInstall(install):
 if is_lepidopter():
 self.update_lepidopter_config()
 
-class ExecutableNotFound(Exception):
-pass
-
-class CreateOoniResources(Command):
-description = ("Create ooni-resources.tar.gz containing test-lists and "
-   "GeoIP data files")
-user_options = []
-
-def initialize_options(self):
-pass
-def finalize_options(self):
-pass
-def download(self, url, directory, filename):
-dst_path = pj(directory, filename)
-args = [
-self.wget,
-"-O",
-dst_path,
-url
-]
-out = run_command(args)
-if out is None:
-raise Exception("Failed to download {0}".format(url))
-return dst_path
-
-def find_executables(self):
-self.wget = find_executable("wget")
-if not self.wget:
-raise ExecutableNotFound("wget")
-self.tar = find_executable("tar")
-if not self.tar:
-raise ExecutableNotFound("tar")
-self.unzip = find_executable("unzip")
-if not self.unzip:
-raise ExecutableNotFound("unzip")
-self.gunzip = find_executable("gunzip")
-if not self.gunzip:
-raise ExecutableNotFound("gunzip")
-
-def run(self):
-dst_path = "dist/ooni-resources.tar.gz"
-
-try:
-self.find_executables()
-except ExecutableNotFound as enf:
-print("ERR: Could not find '{0}'".format(enf.message))
-return
-
-tmp_dir = tempfile.mkdtemp()
-pkg_dir = tempfile.mkdtemp()
-
-os.mkdir(pj(pkg_dir, "resources"))
-os.mkdir(pj(pkg_dir, "GeoIP"))
-
-try:
-geoip_asn_path = self.download(GEOIP_ASN_URL, tmp_dir, 
"GeoIPASNum.dat.gz")
-except Exception as exc:
-print(exc.message)
-return
-try:
-

[tor-commits] [ooni-probe/master] Don't raise an exception if a deck is already enabled

[art]

commit 886e19ec52e96a52c4a1d34d8126337eea18d66c
Author: Arturo Filastò 
Date:   Wed Aug 31 19:42:58 2016 +0200

Don't raise an exception if a deck is already enabled

* Disable debug level logging in web server
---
 ooni/deck/store.py| 6 +-
 ooni/ui/web/server.py | 2 --
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/ooni/deck/store.py b/ooni/deck/store.py
index 295f817..c938d56 100644
--- a/ooni/deck/store.py
+++ b/ooni/deck/store.py
@@ -162,7 +162,11 @@ class DeckStore(object):
 if not deck_path.exists():
 raise DeckNotFound(deck_id)
 deck_enabled_path = self.enabled_directory.child(deck_id + '.yaml')
-deck_path.linkTo(deck_enabled_path)
+try:
+deck_path.linkTo(deck_enabled_path)
+except OSError as ose:
+if ose.errno != errno.EEXIST:
+raise
 
 def disable(self, deck_id):
 deck_enabled_path = self.enabled_directory.child(deck_id + '.yaml')
diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index 2e89370..26bfd47 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -26,8 +26,6 @@ from ooni.measurements import get_summary, get_measurement, 
list_measurements
 from ooni.measurements import MeasurementNotFound, MeasurementInProgress
 from ooni.geoip import probe_ip
 
-config.advanced.debug = True
-
 class WebUIError(Exception):
 def __init__(self, code, message):
 self.code = code



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Bind by default to localhost. Add support for disabling the web UI.

[art]

commit e020064d31ee93e85fda73dc7a89ac55b97f4aa4
Author: Arturo Filastò 
Date:   Fri Aug 26 18:44:53 2016 +0200

Bind by default to localhost. Add support for disabling the web UI.

* Expose these options via configuration options
This fixes #589
---
 ooni/agent/agent.py |  9 +
 ooni/scripts/ooniprobe_agent.py |  6 +++---
 ooni/settings.py| 10 +-
 ooni/ui/web/web.py  |  3 ++-
 4 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/ooni/agent/agent.py b/ooni/agent/agent.py
index 0311cef..01e2162 100644
--- a/ooni/agent/agent.py
+++ b/ooni/agent/agent.py
@@ -14,10 +14,11 @@ class AgentService(service.MultiService):
 self.scheduler_service = SchedulerService(director)
 self.scheduler_service.setServiceParent(self)
 
-self.web_ui_service = WebUIService(director,
-   self.scheduler_service,
-   web_ui_port)
-self.web_ui_service.setServiceParent(self)
+if not config.advanced.disabled_webui:
+self.web_ui_service = WebUIService(director,
+   self.scheduler_service,
+   web_ui_port)
+self.web_ui_service.setServiceParent(self)
 
 
 def startService(self):
diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py
index 30df9b9..08fba89 100644
--- a/ooni/scripts/ooniprobe_agent.py
+++ b/ooni/scripts/ooniprobe_agent.py
@@ -11,14 +11,12 @@ from ooni.utils import log
 from ooni.settings import config
 from ooni.agent.agent import AgentService
 
-WEB_UI_PORT = 8842
-WEB_UI_URL = "http://127.0.0.1:{0}".format(WEB_UI_PORT)
 
 class StartOoniprobeAgentPlugin:
 tapname = "ooniprobe"
 
 def makeService(self, so):
-return AgentService(WEB_UI_PORT)
+return AgentService(config.advanced.webui_port)
 
 class OoniprobeTwistdConfig(twistd.ServerOptions):
 subCommands = [
@@ -70,6 +68,8 @@ def start_agent(options=None):
 "StartOoniprobeAgent": StartOoniprobeAgentPlugin()
 }
 print("Starting ooniprobe agent.")
+WEB_UI_URL = "http://{0}:{1}".format(
+config.advanced.webui_address, config.advanced.webui_port)
 print("To view the GUI go to %s" % WEB_UI_URL)
 log.start()
 twistd.runApp(twistd_config)
diff --git a/ooni/settings.py b/ooni/settings.py
index 3da1da7..2bacd57 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -65,6 +65,11 @@ advanced:
 # insecure_backend: false
 # The preferred backend type, can be one of onion, https or cloudfront
 preferred_backend: {preferred_backend}
+# The port and address for the Web UI
+#webui_port: 8842
+#webui_address: "127.0.0.1"
+# Should the Web UI be disabled
+#disable_webui: false
 tor:
 #socks_port: 8801
 #control_port: 8802
@@ -121,7 +126,10 @@ defaults = {
 "reporting_retries": 5,
 "reporting_concurrency": 7,
 "insecure_backend": False,
-"preferred_backend": "onion"
+"preferred_backend": "onion",
+"webui_port": 8842,
+"webui_address": "127.0.0.1",
+"webui_disabled": False
 },
 "tor": {
 "socks_port": None,
diff --git a/ooni/ui/web/web.py b/ooni/ui/web/web.py
index 10bbef1..f36e0a2 100644
--- a/ooni/ui/web/web.py
+++ b/ooni/ui/web/web.py
@@ -19,7 +19,8 @@ class WebUIService(service.MultiService):
 web_ui_api = WebUIAPI(config, self.director, self.scheduler)
 self._port = reactor.listenTCP(
 self.port_number,
-server.Site(web_ui_api.app.resource())
+server.Site(web_ui_api.app.resource()),
+interface=config.advanced.webui_address
 )
 
 def stopService(self):



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Convert the director status into a property so the IP and ASN get's updated

[art]

commit c7970d24f0285e08a57a3b362889f308dbcdbf3c
Author: Arturo Filastò 
Date:   Mon Aug 1 19:02:23 2016 +0200

Convert the director status into a property so the IP and ASN get's updated

* Make another call to resolveGeodata after we fetch the inputs

(this should fix the issue @agrabeli reported in erroneous IP lookups)
---
 ooni/agent/scheduler.py |  1 +
 ooni/ui/web/server.py   | 22 --
 2 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 0d8689c..a6d689f 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -84,6 +84,7 @@ class UpdateInputsAndResources(ScheduledTask):
 yield probe_ip.lookup()
 yield resources.check_for_update(probe_ip.geodata['countrycode'])
 yield input_store.update(probe_ip.geodata['countrycode'])
+yield probe_ip.resolveGeodata()
 
 
 class UploadReports(ScheduledTask):
diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index 50db3ad..a03daa4 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -127,13 +127,7 @@ class WebUIAPI(object):
 self._xsrf_token = ''.join([rng.choice(token_space)
 for _ in range(30)])
 
-self.status = {
-"software_version": ooniprobe_version,
-"software_name": "ooniprobe",
-"asn": probe_ip.geodata['asn'],
-"country_code": probe_ip.geodata['countrycode'],
-"director_started": False
-}
+self._director_started = False
 
 self.status_poller = LongPoller(
 self._long_polling_timeout, _reactor)
@@ -150,14 +144,22 @@ class WebUIAPI(object):
 d.addCallback(self.director_started)
 d.addBoth(lambda _: self.status_poller.notify())
 
+@property
+def status(self):
+return {
+"software_version": ooniprobe_version,
+"software_name": "ooniprobe",
+"asn": probe_ip.geodata['asn'],
+"country_code": probe_ip.geodata['countrycode'],
+"director_started": self._director_started
+}
+
 def handle_director_event(self, event):
 log.msg("Handling event {0}".format(event.type))
 self.director_event_poller.notify(event)
 
 def director_started(self, _):
-self.status['director_started'] = True
-self.status["asn"] = probe_ip.geodata['asn']
-self.status["country_code"] = probe_ip.geodata['countrycode']
+self._director_started = True
 
 @app.handle_errors(NotFound)
 @xsrf_protect(check=False)



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Bump to alpha.2

[art]

commit f3d3bb10d52e472c880eaa0ca3f69cc3fe14dc6d
Author: Arturo Filastò 
Date:   Mon Aug 8 16:42:16 2016 +0200

Bump to alpha.2

* Use semantic versioning
---
 ooni/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ooni/__init__.py b/ooni/__init__.py
index 1a31608..9eecb67 100644
--- a/ooni/__init__.py
+++ b/ooni/__init__.py
@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 
 __author__ = "Open Observatory of Network Interference"
-__version__ = "2.0.0a1"
+__version__ = "2.0.0-alpha.2"
 
 __all__ = [
 'agent',



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] settings.py: make sure default paths are okay

[art]

commit 8048dbd1d1775dc658537b138e102a9e8319f147
Author: Simone Basso 
Date:   Mon Aug 8 19:24:29 2016 +0200

settings.py: make sure default paths are okay
---
 ooni/settings.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ooni/settings.py b/ooni/settings.py
index 16de94d..3da1da7 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -143,8 +143,8 @@ if hasattr(sys, 'real_prefix'):
 # These are the the embedded settings
 _SETTINGS_INI = os.path.join(OONIPROBE_ROOT, 'settings.ini')
 
-USR_SHARE_PATH = '/var/lib/ooni'
-VAR_LIB_PATH = '/usr/share/ooni'
+USR_SHARE_PATH = '/usr/share/ooni'
+VAR_LIB_PATH = '/var/lib/ooni'
 ETC_PATH = '/etc'
 
 if IS_VIRTUALENV:



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Bundle rebuilt

[art]

commit 9a26fc9c70f781bba30c0680c7ff542e2cab83d8
Author: Arturo Filastò 
Date:   Mon Jul 18 23:09:08 2016 +0200

Bundle rebuilt
---
 ooni/ui/web/client/index.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ooni/ui/web/client/index.html b/ooni/ui/web/client/index.html
index ba9ab37..3812461 100644
--- a/ooni/ui/web/client/index.html
+++ b/ooni/ui/web/client/index.html
@@ -13,5 +13,5 @@
 
   Loading...
 
-  
+  
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Remove debug

[art]

commit 19f2fc8b08bd5a3172388044ee766b6aa75c45d7
Author: Arturo Filastò 
Date:   Mon Jul 18 20:16:55 2016 +0200

Remove debug
---
 ooni/ui/web/server.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index ca51041..e97cd8b 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -94,11 +94,10 @@ class WebUIAPI(object):
 
 def failed_measurement(self, measurement_id, failure):
 del self.status['active_measurements'][measurement_id]
-self.add_failure(failure)
+self.add_failure(str(failure))
 
 @app.route('/api/status', methods=["GET"])
 def api_status(self, request):
-print("Rendering status...")
 return self.render_json(self.status, request)
 
 @app.route('/api/status/update', methods=["GET"])



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Add minimal outline of the ooniprobe-agent and new deck format

[art]

commit 7829363f1066a469995c0410025d7e895522363f
Author: Arturo Filastò 
Date:   Mon Jul 25 16:08:05 2016 +0200

Add minimal outline of the ooniprobe-agent and new deck format

* Use DuckDuckGo to perform geoip lookups instead of torproject.org

* Big refactoring of the Director
---
 ooni/agent/__init__.py |   0
 ooni/agent/agent.py|  21 ++
 ooni/agent/scheduler.py|  27 ++
 ooni/deck.py   | 715 +
 ooni/director.py   | 231 -
 ooni/geoip.py  |   8 +-
 ooni/measurements.py   |  43 ---
 ooni/nettest.py|   3 +-
 ooni/results.py|  39 +++
 ooni/tests/bases.py|   2 +-
 ooni/tests/test_deck.py|  57 +++-
 ooni/tests/test_director.py|  12 +-
 ooni/tests/test_nettest.py |   8 +-
 ooni/tests/test_oonideckgen.py |   7 +-
 ooni/tests/test_oonireport.py  |  18 +-
 ooni/ui/cli.py |   8 +-
 ooni/ui/web/client/index.html  |   2 +-
 ooni/ui/web/server.py  | 203 ++--
 ooni/ui/web/web.py |  56 +---
 ooni/utils/onion.py|  14 -
 20 files changed, 974 insertions(+), 500 deletions(-)

diff --git a/ooni/agent/__init__.py b/ooni/agent/__init__.py
new file mode 100644
index 000..e69de29
diff --git a/ooni/agent/agent.py b/ooni/agent/agent.py
new file mode 100644
index 000..a1394f0
--- /dev/null
+++ b/ooni/agent/agent.py
@@ -0,0 +1,21 @@
+from twisted.application import service
+from ooni.director import Director
+from ooni.settings import config
+
+from ooni.ui.web.web import WebUIService
+from ooni.agent.scheduler import SchedulerService
+
+class AgentService(service.MultiService):
+def __init__(self):
+service.MultiService.__init__(self)
+
+director = Director()
+config.set_paths()
+config.initialize_ooni_home()
+config.read_config_file()
+
+self.web_ui_service = WebUIService(director)
+self.web_ui_service.setServiceParent(self)
+
+self.scheduler_service = SchedulerService(director)
+self.scheduler_service.setServiceParent(self)
diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
new file mode 100644
index 000..1004597
--- /dev/null
+++ b/ooni/agent/scheduler.py
@@ -0,0 +1,27 @@
+from twisted.application import service
+from twisted.internet import task
+
+class SchedulerService(service.MultiService):
+"""
+This service is responsible for running the periodic tasks.
+"""
+def __init__(self, director, interval=30):
+service.MultiService.__init__(self)
+self.director = director
+self.interval = interval
+self._looping_call = task.LoopingCall(self._should_run)
+
+def _should_run(self):
+"""
+This function is called every self.interval seconds to check
+which periodic tasks should be run.
+"""
+pass
+
+def startService(self):
+service.MultiService.startService(self)
+self._looping_call.start(self.interval)
+
+def stopService(self):
+service.MultiService.stopService(self)
+self._looping_call.stop()
diff --git a/ooni/deck.py b/ooni/deck.py
index 9f17530..1746d26 100644
--- a/ooni/deck.py
+++ b/ooni/deck.py
@@ -1,27 +1,28 @@
 # -*- coding: utf-8 -*-
-import csv
 import os
-import yaml
+import csv
 import json
 
+from copy import deepcopy
 from hashlib import sha256
-from datetime import datetime
-from ooni.backend_client import CollectorClient, BouncerClient
-from ooni.backend_client import WebConnectivityClient, guess_backend_type
-from ooni.nettest import NetTestLoader
-from ooni.settings import config
 
-from ooni.otime import timestampNowISO8601UTC
+import yaml
 
-from ooni.resources.update import check_for_update
+from twisted.internet import defer
+from twisted.python.filepath import FilePath
 
-from ooni.utils import log
 from ooni import constants
 from ooni import errors as e
+from ooni.backend_client import CollectorClient, BouncerClient
+from ooni.backend_client import WebConnectivityClient, guess_backend_type
+from ooni.nettest import NetTestLoader
+from ooni.otime import timestampNowISO8601UTC
+from ooni.resources import check_for_update
+from ooni.settings import config
+from ooni.utils import generate_filename
+from ooni.utils import log
 
-from twisted.python.filepath import FilePath
-from twisted.internet import defer
-
+from ooni.results import generate_summary
 
 class InputFile(object):
 def __init__(self, input_hash, base_path=config.inputs_directory):
@@ -116,6 +117,25 @@ def nettest_to_path(path, allow_arbitrary_paths=False):
 return found_path
 
 
+def get_preferred_bouncer():
+preferred_backend = config.advanced.get(
+"preferred_backend", "onion"
+)
+bouncer_address = getattr(
+constants, "CANONICAL_BOUNCER_{0}".format(
+preferred_backend.upper()
+)
+ 

[tor-commits] [ooni-probe/master] [Web UI] Refactoring of web UI

[art]

commit bd4458d07a7f6755b5c104c068e9c8dd86ce2852
Author: Arturo Filastò 
Date:   Mon Jun 13 00:44:58 2016 +0200

[Web UI] Refactoring of web UI

* Move into the ooni/ui tree
* Move the web UI related files to a separate repository
---
 ooni/ui/web/server.py |   3 +
 ooni/web/resources.py | 172 --
 ooni/web/root.py  |  40 
 ooni/webui.py |  57 -
 4 files changed, 3 insertions(+), 269 deletions(-)

diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index 3da6f58..5be581c 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -197,4 +197,7 @@ class WebUIAPI(object):
 path = rpath("build")
 print(path)
 return static.File(path)
+<<< acda284b56fa3a75acbe7d000fbdefb643839948
 
+===
+>>> [Web UI] Refactoring of web UI
diff --git a/ooni/web/resources.py b/ooni/web/resources.py
deleted file mode 100644
index 336fdc0..000
--- a/ooni/web/resources.py
+++ /dev/null
@@ -1,172 +0,0 @@
-import json
-from twisted.web import resource
-from twisted.python import usage
-
-from ooni import errors
-from ooni.nettest import NetTestLoader
-
-
-class WuiResource(resource.Resource):
-isLeaf = True
-XSRF_HEADER = 'X-XSRF-TOKEN'
-# XXX set this to true when stable version
-XSRF_PROTECTION = False
-
-def __init__(self, director):
-self.director = director
-resource.Resource.__init__(self)
-
-def check_xsrf(self, request):
-if self.XSRF_PROTECTION is False:
-return True
-if request.requestHeaders.hasHeader(self.XSRF_HEADER):
-return True
-return False
-
-def render(self, request):
-if not self.check_xsrf(request):
-obj = {
-'error_code': 400,
-'error_message': ('Missing cross site request forgery '
-  'header \'{}\''.format(self.XSRF_HEADER))
-}
-request.setResponseCode(403)
-return self.render_json(obj, request)
-obj = resource.Resource.render(self, request)
-return self.render_json(obj, request)
-
-def render_json(self, obj, request):
-json_string = json.dumps(obj) + "\n"
-request.setHeader('Content-Type', 'application/json')
-request.setHeader('Content-Length', len(json_string))
-return json_string
-
-
-class DecksGenerate(WuiResource):
-def render_GET(self, request):
-return {"generate": "deck"}
-
-
-class DecksStart(WuiResource):
-def __init__(self, director, deck_name):
-WuiResource.__init__(self, director)
-self.deck_name = deck_name
-
-def render_GET(self, request):
-return {"start": self.deck_name}
-
-
-class DecksStop(WuiResource):
-def __init__(self, director, deck_id):
-WuiResource.__init__(self, director)
-self.deck_id = deck_id
-
-def render_GET(self, request):
-return {"stop": self.deck_id}
-
-
-class DecksStatus(WuiResource):
-def __init__(self, director, deck_name):
-WuiResource.__init__(self, director)
-self.deck_name = deck_name
-
-def render_GET(self, request):
-return {"deck": self.deck_name}
-
-
-class DecksList(WuiResource):
-def render_GET(self, request):
-return {"deck": "list"}
-
-
-def getNetTestLoader(test_options, test_file):
-"""
-Args:
-test_options: (dict) containing as keys the option names.
-
-test_file: (string) the path to the test_file to be run.
-Returns:
-an instance of :class:`ooni.nettest.NetTestLoader` with the specified
-test_file and the specified options.
-"""
-options = []
-for k, v in test_options.items():
-options.append('--'+k)
-options.append(v)
-
-net_test_loader = NetTestLoader(options,
-test_file=test_file)
-return net_test_loader
-
-class TestsStart(WuiResource):
-def __init__(self, director, test_name):
-WuiResource.__init__(self, director)
-self.test_name = test_name
-
-def render_POST(self, request):
-try:
-net_test = self.director.netTests[self.test_name]
-except KeyError:
-request.setResponseCode(500)
-return {
-'error_code': 500,
-'error_message': 'Could not find the specified test'
-}
-test_options = json.load(request.content)
-net_test_loader = getNetTestLoader(test_options, net_test['path'])
-try:
-net_test_loader.checkOptions()
-# XXX we actually want to generate the report_filename in a smart
-# way so that we can know where it is located and learn the results
-# of the measurement.
-report_filename = None
-self.director.startNetTest(net_test_loader, report_filename)
-except 

[tor-commits] [ooni-probe/master] Start outlining update mechanism for inputs

[art]

commit cee9d833edca688429e5323426a1b9ef43050d46
Author: Arturo Filastò 
Date:   Fri Jul 22 19:49:41 2016 +0200

Start outlining update mechanism for inputs

* Expose only simple options in web UI
---
 ooni/deck.py   | 109 +--
 ooni/director.py   |   5 ++
 ooni/nettest.py|  34 +---
 ooni/nettests/blocking/web_connectivity.py |   6 ++
 ooni/otime.py  |   7 ++
 ooni/resources/__init__.py |  19 +++-
 ooni/resources/update.py   | 134 -
 ooni/settings.py   |   2 +-
 ooni/ui/cli.py |   1 -
 ooni/ui/web/client/index.html  |   2 +-
 10 files changed, 294 insertions(+), 25 deletions(-)

diff --git a/ooni/deck.py b/ooni/deck.py
index 1e1d580..9f17530 100644
--- a/ooni/deck.py
+++ b/ooni/deck.py
@@ -1,21 +1,27 @@
 # -*- coding: utf-8 -*-
+import csv
+import os
+import yaml
+import json
 
+from hashlib import sha256
+from datetime import datetime
 from ooni.backend_client import CollectorClient, BouncerClient
 from ooni.backend_client import WebConnectivityClient, guess_backend_type
 from ooni.nettest import NetTestLoader
 from ooni.settings import config
-from ooni.utils import log, onion
+
+from ooni.otime import timestampNowISO8601UTC
+
+from ooni.resources.update import check_for_update
+
+from ooni.utils import log
 from ooni import constants
 from ooni import errors as e
 
 from twisted.python.filepath import FilePath
 from twisted.internet import defer
 
-import os
-import yaml
-import json
-from hashlib import sha256
-
 
 class InputFile(object):
 def __init__(self, input_hash, base_path=config.inputs_directory):
@@ -411,3 +417,94 @@ class Deck(InputFile):
 raise e.UnableToLoadDeckInput
 
 i['test_options'][i['key']] = input_file.cached_file
+
+
+class InputStore(object):
+def __init__(self):
+self.path = FilePath(config.inputs_directory)
+self.resources = FilePath(config.resources_directory)
+
+@defer.inlineCallbacks
+def update_url_lists(self, country_code):
+countries = ["global"]
+if country_code == "ZZ":
+country_code = None
+else:
+countries.append(country_code)
+
+for cc in countries:
+in_file = 
self.resources.child("citizenlab-test-lists").child("{0}.csv".format(cc))
+if not in_file.exists():
+yield check_for_update(country_code)
+
+if not in_file.exists():
+continue
+
+# XXX maybe move this to some utility function.
+# It's duplicated in oonideckgen.
+data_fname = "citizenlab-test-lists_{0}.txt".format(cc)
+desc_fname = "citizenlab-test-lists_{0}.desc".format(cc)
+
+out_file = self.path.child("data").child(data_fname)
+out_fh = out_file.open('w')
+with in_file.open('r') as in_fh:
+csvreader = csv.reader(in_fh)
+csvreader.next()
+for row in csvreader:
+out_fh.write("%s\n" % row[0])
+out_fh.close()
+
+desc_file = self.path.child("descriptors").child(desc_fname)
+with desc_file.open('w') as out_fh:
+if cc == "global":
+name = "List of globally accessed websites"
+else:
+# XXX resolve this to a human readable country name
+country_name = cc
+name = "List of websites for {0}".format(country_name)
+json.dump({
+"name": name,
+"filepath": out_file.path,
+"last_updated": timestampNowISO8601UTC(),
+"id": "citizenlab_test_lists_{0}_txt".format(cc),
+"type": "file/url"
+}, out_fh)
+
+@defer.inlineCallbacks
+def create(self, country_code=None):
+self.path.child("descriptors").makedirs(ignoreExistingDirectory=True)
+self.path.child("data").makedirs(ignoreExistingDirectory=True)
+yield self.update_url_lists(country_code)
+
+@defer.inlineCallbacks
+def update(self, country_code=None):
+yield self.update_url_lists(country_code)
+
+def list(self):
+inputs = []
+descs = self.path.child("descriptors")
+if not descs.exists():
+return inputs
+
+for fn in descs.listdir():
+with descs.child(fn).open("r") as in_fh:
+inputs.append(json.load(in_fh))
+return inputs
+
+class DeckStore(object):
+def __init__(self):
+self.path = FilePath(config.decks_directory)
+
+def update(self):
+pass
+
+def get(self):
+pass
+
+class NGInput(object):
+def __init__(self, 

[tor-commits] [ooni-probe/master] Write ooniprobe reports in JSON format on disk

[art]

commit 1ec88b611d77048b8281d3358b20883388bd8283
Author: Arturo Filastò 
Date:   Thu Jul 14 21:30:01 2016 +0200

Write ooniprobe reports in JSON format on disk

* Implement various API endpoints
---
 ooni/measurements.py   |  19 +++
 ooni/reporter.py   | 116 
 ooni/settings.py   |   3 +-
 ooni/ui/web/server.py  | 142 +++--
 ooni/ui/web/web.py |   7 ++-
 ooni/utils/__init__.py |   6 ++-
 6 files changed, 179 insertions(+), 114 deletions(-)

diff --git a/ooni/measurements.py b/ooni/measurements.py
index 5244ea4..976b125 100644
--- a/ooni/measurements.py
+++ b/ooni/measurements.py
@@ -9,26 +9,27 @@ class GenerateResults(object):
 self.input_file = input_file
 
 def process_web_connectivity(self, entry):
-anomaly = {}
-anomaly['result'] = False
+result = {}
+result['anomaly'] = False
 if entry['test_keys']['blocking'] is not False:
-anomaly['result'] = True
-anomaly['url'] = entry['input']
-return anomaly
+result['anomaly'] = True
+result['url'] = entry['input']
+return result
 
 def output(self, output_file):
 results = {}
 with open(self.input_file) as in_file:
-for line in in_file:
+for idx, line in enumerate(in_file):
 entry = json.loads(line.strip())
 if entry['test_name'] not in self.supported_tests:
 raise Exception("Unsupported test")
-anomaly = getattr(self, 'process_'+entry['test_name'])(entry)
+result = getattr(self, 'process_'+entry['test_name'])(entry)
+result['idx'] = idx
 results['test_name'] = entry['test_name']
 results['country_code'] = entry['probe_cc']
 results['asn'] = entry['probe_asn']
-results['anomalies'] = results.get('anomalies', [])
-results['anomalies'].append(anomaly)
+results['results'] = results.get('results', [])
+results['results'].append(result)
 
 with open(output_file, "w") as fw:
 json.dump(results, fw)
diff --git a/ooni/reporter.py b/ooni/reporter.py
index f76fada..f07b3cf 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -1,5 +1,6 @@
 import uuid
 import yaml
+import json
 import os
 
 from copy import deepcopy
@@ -206,6 +207,56 @@ class YAMLReporter(OReporter):
 def finish(self):
 self._stream.close()
 
+class NJSONReporter(OReporter):
+
+"""
+report_destination:
+the destination directory of the report
+
+"""
+
+def __init__(self, test_details, report_filename):
+self.report_path = report_filename
+OReporter.__init__(self, test_details)
+
+def _writeln(self, line):
+self._write("%s\n" % line)
+
+def _write(self, data):
+if not self._stream:
+raise errors.ReportNotCreated
+if self._stream.closed:
+raise errors.ReportAlreadyClosed
+s = str(data)
+assert isinstance(s, type(''))
+self._stream.write(s)
+untilConcludes(self._stream.flush)
+
+def writeReportEntry(self, entry):
+if isinstance(entry, Measurement):
+e = deepcopy(entry.testInstance.report)
+elif isinstance(entry, dict):
+e = deepcopy(entry)
+else:
+raise Exception("Failed to serialise entry")
+report_entry = {
+'input': e.pop('input', None),
+'id': str(uuid.uuid4()),
+'test_start_time': e.pop('test_start_time', None),
+'measurement_start_time': e.pop('measurement_start_time', None),
+'test_runtime': e.pop('test_runtime', None),
+'test_keys': e
+}
+report_entry.update(self.testDetails)
+self._write(json.dumps(report_entry))
+self._write("\n")
+
+def createReport(self):
+self._stream = open(self.report_path, 'w+')
+
+def finish(self):
+self._stream.close()
+
 
 class OONIBReporter(OReporter):
 
@@ -219,25 +270,20 @@ class OONIBReporter(OReporter):
 def serializeEntry(self, entry, serialisation_format="yaml"):
 if serialisation_format == "json":
 if isinstance(entry, Measurement):
-report_entry = {
-'input': entry.testInstance.report.pop('input', None),
-'id': str(uuid.uuid4()),
-'test_start_time': 
entry.testInstance.report.pop('test_start_time', None),
-'measurement_start_time': 
entry.testInstance.report.pop('measurement_start_time', None),
-'test_runtime': 
entry.testInstance.report.pop('test_runtime', None),
-'test_keys': entry.testInstance.report
-}
+e = 

[tor-commits] [ooni-probe/master] Better handling of exceptions inside of web ui server

[art]

commit 08ed96f1878a7d1013782553089c05f058d85fab
Author: Arturo Filastò 
Date:   Fri Jul 22 19:49:08 2016 +0200

Better handling of exceptions inside of web ui server
---
 ooni/ui/web/server.py | 145 +-
 1 file changed, 84 insertions(+), 61 deletions(-)

diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index e97cd8b..e63c08f 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -45,6 +45,12 @@ def getNetTestLoader(test_options, test_file):
 test_file=test_file)
 return net_test_loader
 
+
+class WebUIError(Exception):
+def __init__(self, code, message):
+self.code = code
+self.message = message
+
 class WebUIAPI(object):
 app = Klein()
 # Maximum number in seconds after which to return a result even if not
@@ -55,6 +61,9 @@ class WebUIAPI(object):
 def __init__(self, config, director):
 self.director = director
 self.config = config
+self.measurement_path = FilePath(config.measurements_directory)
+self.decks_path = FilePath(config.decks_directory)
+
 self.status = {
 "software_version": ooniprobe_version,
 "software_name": "ooniprobe",
@@ -73,7 +82,7 @@ class WebUIAPI(object):
 d.addBoth(lambda _: self.broadcast_status_update())
 
 def add_failure(self, failure):
-self.status['failures'].append(failure)
+self.status['failures'].append(str(failure))
 
 def director_started(self, _):
 self.status['director_started'] = True
@@ -91,11 +100,39 @@ class WebUIAPI(object):
 def completed_measurement(self, measurement_id):
 del self.status['active_measurements'][measurement_id]
 self.status['completed_measurements'].append(measurement_id)
+measurement_dir = self.measurement_path.child(measurement_id)
+
+measurement = measurement_dir.child('measurements.njson.progress')
+
+# Generate the summary.json file
+summary = measurement_dir.child('summary.json')
+gr = GenerateResults(measurement.path)
+gr.output(summary.path)
+
+measurement.moveTo(measurement_dir.child('measurements.njson'))
 
 def failed_measurement(self, measurement_id, failure):
 del self.status['active_measurements'][measurement_id]
 self.add_failure(str(failure))
 
+@app.handle_errors(NotFound)
+def not_found(self, request, _):
+request.redirect('/client/')
+
+@app.handle_error(WebUIError)
+def web_ui_error(self, request, error):
+request.setResponseCode(error.code)
+return self.render_json({
+"error_code": error.code,
+"error_message": error.message
+}, request)
+
+def render_json(self, obj, request):
+json_string = json.dumps(obj) + "\n"
+request.setHeader('Content-Type', 'application/json')
+request.setHeader('Content-Length', len(json_string))
+return json_string
+
 @app.route('/api/status', methods=["GET"])
 def api_status(self, request):
 return self.render_json(self.status, request)
@@ -115,16 +152,6 @@ class WebUIAPI(object):
 
 return status_update
 
-@app.handle_errors(NotFound)
-def not_found(self, request, _):
-request.redirect('/client/')
-
-def render_json(self, obj, request):
-json_string = json.dumps(obj) + "\n"
-request.setHeader('Content-Type', 'application/json')
-request.setHeader('Content-Length', len(json_string))
-return json_string
-
 @app.route('/api/deck/generate', methods=["GET"])
 def api_deck_generate(self, request):
 return self.render_json({"generate": "deck"}, request)
@@ -135,6 +162,9 @@ class WebUIAPI(object):
 
 @app.route('/api/deck', methods=["GET"])
 def api_deck_list(self, request):
+for deck_id in self.decks_path.listdir():
+pass
+
 return self.render_json({"command": "deck-list"}, request)
 
 @defer.inlineCallbacks
@@ -146,13 +176,12 @@ class WebUIAPI(object):
 test_details = net_test_loader.getTestDetails()
 measurement_id = generate_filename(test_details)
 
-measurement_dir = os.path.join(
-config.measurements_directory,
-measurement_id
-)
-os.mkdir(measurement_dir)
-report_filename = os.path.join(measurement_dir,
-   "measurements.njson")
+measurement_dir = self.measurement_path.child(measurement_id)
+measurement_dir.createDirectory()
+
+report_filename = measurement_dir.child(
+"measurements.njson.progress").path
+
 measurement_ids.append(measurement_id)
 self.status['active_measurements'][measurement_id] = {
 'test_name': test_details['test_name'],
@@ -160,7 +189,8 @@ class WebUIAPI(object):
 

[tor-commits] [ooni-probe/master] Start the director after the GUI is available. Implement a long polling mechanism.

[art]

commit f7f272d93a640d091855c7ca85c274b3aa7113e9
Author: Arturo Filastò 
Date:   Mon Jul 18 19:18:39 2016 +0200

Start the director after the GUI is available. Implement a long polling 
mechanism.
---
 ooni/ui/web/server.py | 80 +--
 ooni/ui/web/web.py| 14 ++---
 2 files changed, 75 insertions(+), 19 deletions(-)

diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index ccc5d87..ca51041 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -3,7 +3,7 @@ from __future__ import print_function
 import os
 import json
 
-from twisted.internet import defer
+from twisted.internet import defer, task, reactor
 from twisted.python import usage
 from twisted.python.filepath import FilePath, InsecurePath
 from twisted.web import static
@@ -11,6 +11,7 @@ from twisted.web import static
 from klein import Klein
 from werkzeug.exceptions import NotFound
 
+from ooni import __version__ as ooniprobe_version
 from ooni import errors
 from ooni.deck import Deck
 from ooni.settings import config
@@ -46,11 +47,74 @@ def getNetTestLoader(test_options, test_file):
 
 class WebUIAPI(object):
 app = Klein()
+# Maximum number in seconds after which to return a result even if not
+# change happenned.
+_long_polling_timeout = 5
+_reactor = reactor
 
 def __init__(self, config, director):
 self.director = director
 self.config = config
-self.active_measurements = {}
+self.status = {
+"software_version": ooniprobe_version,
+"software_name": "ooniprobe",
+"asn": config.probe_ip.geodata['asn'],
+"country_code": config.probe_ip.geodata['countrycode'],
+"active_measurements": {},
+"completed_measurements": [],
+"director_started": False,
+"failures": []
+}
+self.status_updates = []
+d = self.director.start(start_tor=True)
+
+d.addCallback(self.director_started)
+d.addErrback(self.director_startup_failed)
+d.addBoth(lambda _: self.broadcast_status_update())
+
+def add_failure(self, failure):
+self.status['failures'].append(failure)
+
+def director_started(self, _):
+self.status['director_started'] = True
+self.status["asn"] = config.probe_ip.geodata['asn']
+self.status["country_code"] = config.probe_ip.geodata['countrycode']
+
+def director_startup_failed(self, failure):
+self.add_failure(failure)
+
+def broadcast_status_update(self):
+for su in self.status_updates:
+if not su.called:
+su.callback(None)
+
+def completed_measurement(self, measurement_id):
+del self.status['active_measurements'][measurement_id]
+self.status['completed_measurements'].append(measurement_id)
+
+def failed_measurement(self, measurement_id, failure):
+del self.status['active_measurements'][measurement_id]
+self.add_failure(failure)
+
+@app.route('/api/status', methods=["GET"])
+def api_status(self, request):
+print("Rendering status...")
+return self.render_json(self.status, request)
+
+@app.route('/api/status/update', methods=["GET"])
+def api_status_update(self, request):
+status_update = defer.Deferred()
+status_update.addCallback(lambda _:
+  self.status_updates.remove(status_update))
+status_update.addCallback(lambda _: self.api_status(request))
+
+self.status_updates.append(status_update)
+
+# After long_polling_timeout we fire the callback
+task.deferLater(self._reactor, self._long_polling_timeout,
+status_update.callback, None)
+
+return status_update
 
 @app.handle_errors(NotFound)
 def not_found(self, request, _):
@@ -91,11 +155,15 @@ class WebUIAPI(object):
 report_filename = os.path.join(measurement_dir,
"measurements.njson")
 measurement_ids.append(measurement_id)
-self.active_measurements[measurement_id] = {
+self.status['active_measurements'][measurement_id] = {
 'test_name': test_details['test_name'],
 'test_start_time': test_details['test_start_time']
 }
-self.director.startNetTest(net_test_loader, report_filename)
+self.broadcast_status_update()
+d = self.director.startNetTest(net_test_loader, report_filename)
+d.addCallback(lambda _: self.completed_measurement(measurement_id))
+d.addErrback(lambda failure:
+ self.failed_measurement(measurement_id, failure))
 
 @app.route('/api/nettest//start', methods=["POST"])
 def api_nettest_start(self, request, test_name):
@@ -147,10 +215,6 @@ class WebUIAPI(object):
 def api_nettest_list(self, request):

[tor-commits] [ooni-probe/master] Call setupBaseAddress after setting the settings

[art]

commit 503c8412828725871be129b073c005a10404df6d
Author: Arturo Filastò 
Date:   Tue Jun 21 18:00:41 2016 +0200

Call setupBaseAddress after setting the settings

This makes it possible to re-instantiate a BackendClient from the settings
---
 ooni/backend_client.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ooni/backend_client.py b/ooni/backend_client.py
index 1039124..506aba4 100644
--- a/ooni/backend_client.py
+++ b/ooni/backend_client.py
@@ -42,7 +42,6 @@ class OONIBClient(object):
 self.backend_type = guess_backend_type(self.base_address)
 self.backend_type = self.backend_type.encode('ascii')
 
-self._setupBaseAddress()
 self.settings = {
 'type': self.backend_type,
 'address': self.base_address,



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Fixes to support cloudfronting and https collectors

[art]

commit 409f8a595ec2b67f89d6fca0f621530b732cedc1
Author: Arturo Filastò 
Date:   Tue Jun 21 16:28:18 2016 +0200

Fixes to support cloudfronting and https collectors
---
 ooni/backend_client.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ooni/backend_client.py b/ooni/backend_client.py
index 506aba4..1039124 100644
--- a/ooni/backend_client.py
+++ b/ooni/backend_client.py
@@ -42,6 +42,7 @@ class OONIBClient(object):
 self.backend_type = guess_backend_type(self.base_address)
 self.backend_type = self.backend_type.encode('ascii')
 
+self._setupBaseAddress()
 self.settings = {
 'type': self.backend_type,
 'address': self.base_address,



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Delete the app component template

[art]

commit d8e04435e4f154466fa9354ce215fff2c7af77d9
Author: Arturo Filastò 
Date:   Mon Jul 18 19:19:24 2016 +0200

Delete the app component template
---
 data/component-template/templ.component.js  | 10 --
 data/component-template/templ.controller.js |  5 -
 data/component-template/templ.css   |  0
 data/component-template/templ.html  |  8 
 data/component-template/templ.js| 19 ---
 5 files changed, 42 deletions(-)

diff --git a/data/component-template/templ.component.js 
b/data/component-template/templ.component.js
deleted file mode 100644
index 10c3c88..000
--- a/data/component-template/templ.component.js
+++ /dev/null
@@ -1,10 +0,0 @@
-var template = require("./{{name}}.html");
-var controller = require("./{{name}}.controller");
-require("./{{name}}.css");
-
-module.exports = {
-  restrict: 'E',
-  bindings: {},
-  template: template,
-  controller: controller
-};
diff --git a/data/component-template/templ.controller.js 
b/data/component-template/templ.controller.js
deleted file mode 100644
index aaf9110..000
--- a/data/component-template/templ.controller.js
+++ /dev/null
@@ -1,5 +0,0 @@
-function {{nameUpper}}Controller() {
-  this.example = "bar";
-}
-
-module.exports = {{nameUpper}}Controller;
diff --git a/data/component-template/templ.css 
b/data/component-template/templ.css
deleted file mode 100644
index e69de29..000
diff --git a/data/component-template/templ.html 
b/data/component-template/templ.html
deleted file mode 100644
index 6b047b9..000
--- a/data/component-template/templ.html
+++ /dev/null
@@ -1,8 +0,0 @@
-
-
-
-
-  
-{{nameUpper}}
-  
-
diff --git a/data/component-template/templ.js b/data/component-template/templ.js
deleted file mode 100644
index 75ee1cf..000
--- a/data/component-template/templ.js
+++ /dev/null
@@ -1,19 +0,0 @@
-var angular = require("angular");
-var uiRouter = require("angular-ui-router");
-var {{name}}Component = require("./{{name}}.component");
-
-var {{name}}Module = angular.module("{{name}}", [
-  uiRouter
-])
-.config(function($stateProvider, $urlRouterProvider){
-
-  $stateProvider.state('{{name}}', {
-url: '/{{name}}',
-template: '<{{name}}>'
-  });
-
-})
-.component("{{name}}", {{name}}Component)
-.name;
-
-module.exports = {{name}}Module;



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Use the testing https and cloudfronted bouncer

[art]

commit acda284b56fa3a75acbe7d000fbdefb643839948
Author: Arturo Filastò 
Date:   Fri Jun 24 16:29:48 2016 +0200

Use the testing https and cloudfronted bouncer

* Bump version number to 1.6.0rc1
---
 ooni/constants.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/ooni/constants.py b/ooni/constants.py
index 6efa52e..38f85b4 100644
--- a/ooni/constants.py
+++ b/ooni/constants.py
@@ -1,5 +1,7 @@
 CANONICAL_BOUNCER_ONION = 'httpo://nkvphnp3p6agi5qq.onion'
-CANONICAL_BOUNCER_HTTPS = 'https://bouncer.ooni.io'
+# XXX Change these two to the production ones once the release is made.
+# CANONICAL_BOUNCER_HTTPS = 'https://bouncer.ooni.io'
+CANONICAL_BOUNCER_HTTPS = 'https://bouncer.test.ooni.io'
 CANONICAL_BOUNCER_CLOUDFRONT = (
 'https://d3kr4emv7f56qa.cloudfront.net/',
 'a0.awsstatic.com'



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Merge branch 'master' of github.com:TheTorProject/ooni-probe

[art]

commit df0214e8ced0be1ab6bf965e1385cbe428ff1e8d
Merge: 7eae5c1 5e86276
Author: Arturo Filastò 
Date:   Fri Jul 29 13:30:01 2016 +0200

Merge branch 'master' of github.com:TheTorProject/ooni-probe

* 'master' of github.com:TheTorProject/ooni-probe:
  Add backend to required options to show error when it's not provided

 ooni/nettests/manipulation/traceroute.py |  5 +++--
 ooni/templates/scapyt.py | 10 +-
 2 files changed, 8 insertions(+), 7 deletions(-)



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] ooni/agent/scheduler.py: fix easter egg date

[art]

commit 1b9cb5e63687ca41abadb38282717b63620c719b
Author: Simone Basso 
Date:   Mon Sep 19 12:43:07 2016 +0200

ooni/agent/scheduler.py: fix easter egg date
---
 ooni/agent/scheduler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index e8fef53..be141eb 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -53,7 +53,7 @@ class FileSystemlockAndMutex(object):
 
 # We use this date to indicate that the scheduled task has never run.
 # Easter egg, try to see what is special about this date :)?
-CANARY_DATE = datetime(1957, 8, 4, tzinfo=tz.tzutc())
+CANARY_DATE = datetime(1957, 10, 4, tzinfo=tz.tzutc())
 
 
 class DidNotRun(Exception):



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] ui/web/server.py: pass binary, not unicode cookies

[art]

commit 21548e9f0f53100336ef4f22d423f5f728a117e5
Author: Simone Basso 
Date:   Mon Sep 19 11:04:53 2016 +0200

ui/web/server.py: pass binary, not unicode cookies

It seems that Twisted has added a function enforcing binary
data (not unicode, or str) when sending to the network.

Thus, make sure that we pass cookies as binary (more specifically
ascii encoded data) to avoid a Twisted error.
---
 ooni/ui/web/server.py | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index eec82d3..385bf18 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -56,13 +56,13 @@ def xsrf_protect(check=True):
 @wraps(f)
 def wrapper(instance, request, *a, **kw):
 should_check = check and instance._enable_xsrf_protection
-token_cookie = request.getCookie(u'XSRF-TOKEN')
+token_cookie = request.getCookie(b'XSRF-TOKEN')
 token_header = request.getHeader(b"X-XSRF-TOKEN")
 if (token_cookie != instance._xsrf_token and
 instance._enable_xsrf_protection):
-request.addCookie(u'XSRF-TOKEN',
+request.addCookie(b'XSRF-TOKEN',
   instance._xsrf_token,
-  path=u'/')
+  path=b'/')
 if should_check and token_cookie != token_header:
 raise WebUIError(404, "Invalid XSRF token")
 return f(instance, request, *a, **kw)
@@ -161,7 +161,7 @@ class WebUIAPI(object):
 # We use a double submit token to protect against XSRF
 rng = SystemRandom()
 token_space = string.letters+string.digits
-self._xsrf_token = ''.join([rng.choice(token_space)
+self._xsrf_token = b''.join([rng.choice(token_space)
 for _ in range(30)])
 
 self._director_started = False



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Add changelog entry for 2.0.0-rc.3

[art]

commit 77c2e0f7f2c1df0d7490d560103ba241c5502c87
Author: Arturo Filastò 
Date:   Mon Sep 19 14:11:00 2016 +0200

Add changelog entry for 2.0.0-rc.3
---
 ChangeLog.rst | 5 +
 1 file changed, 5 insertions(+)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 3c0ecd8..f847675 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,6 +1,11 @@
 Changelog
 =
 
+v2.0.0-rc.3 (Mon, 19 Sep 2016)
+--
+
+Bugfixing and code cleanup
+
 v2.0.0-rc.2 (Tue, 13 Sep 2016)
 --
 

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Retry 3 times the http_requests integration tests.

[art]

commit 7be2fc5571c9a0c7fa65de70381819f9f458e6fe
Author: Arturo Filastò 
Date:   Fri Sep 16 19:19:31 2016 +0200

Retry 3 times the http_requests integration tests.

Hopefully this will overcome the transient failures.
---
 ooni/tests/test_oonicli.py | 13 +
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py
index 4a6edf9..05bc273 100644
--- a/ooni/tests/test_oonicli.py
+++ b/ooni/tests/test_oonicli.py
@@ -111,6 +111,7 @@ class TestRunDirector(ConfigTestCase):
 
 @defer.inlineCallbacks
 def test_http_requests(self):
+retries = 3
 def verify_function(entry):
 assert 'body_length_match' in entry
 assert 'body_proportion' in entry
@@ -119,10 +120,14 @@ class TestRunDirector(ConfigTestCase):
 assert 'factor' in entry
 assert 'headers_diff' in entry
 assert 'headers_match' in entry
-
-yield self.run_helper('blocking/http_requests',
-  ['-u', 'http://torproject.org/'],
-  verify_function)
+while retries > 0:
+try:
+yield self.run_helper('blocking/http_requests',
+['-u', 'http://torproject.org/'],
+verify_function)
+break
+except:
+retries -= 1
 
 @defer.inlineCallbacks
 def test_http_requests_with_file(self):



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Strip the filepath from input_store listing

[art]

commit baa505da2dcd2cf69e58fe212a2bc8478cf99617
Author: Arturo Filastò 
Date:   Sun Sep 18 16:41:38 2016 +0200

Strip the filepath from input_store listing
---
 ooni/deck/store.py| 4 ++--
 ooni/ui/web/server.py | 6 --
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/ooni/deck/store.py b/ooni/deck/store.py
index f30b0d6..3e20e01 100644
--- a/ooni/deck/store.py
+++ b/ooni/deck/store.py
@@ -115,7 +115,7 @@ class InputStore(object):
 if self._cache_stale:
 self._update_cache()
 try:
-input_desc = self._cache[input_id]
+input_desc = deepcopy(self._cache[input_id])
 except KeyError:
 raise InputNotFound(input_id)
 return input_desc
@@ -190,7 +190,7 @@ class DeckStore(object):
 if self._cache_stale:
 self._update_cache()
 try:
-return self._cache[deck_id]
+return deepcopy(self._cache[deck_id])
 except KeyError:
 raise DeckNotFound(deck_id)
 
diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index cee42ce..eec82d3 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -474,7 +474,7 @@ class WebUIAPI(object):
 
 except errors.InsufficientPrivileges:
 raise WebUIError(
-400, 'Insufficient priviledges'
+400, 'Insufficient privileges'
 )
 except:
 raise WebUIError(
@@ -511,8 +511,10 @@ class WebUIAPI(object):
 @xsrf_protect(check=False)
 @requires_true(attrs=['_is_initialized'])
 def api_input_details(self, request, input_id):
+input_desc = self.director.input_store.get(input_id)
+input_desc.pop('filepath')
 return self.render_json(
-self.director.input_store.get(input_id), request
+input_desc, request
 )
 
 @app.route('/api/measurement', methods=["GET"])



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Various code style fixes suggested by @bassosimone

[art]

commit 3483c8005ecf8eb0c856f04913f6132f547f2d84
Author: Arturo Filastò 
Date:   Fri Sep 16 11:56:38 2016 +0200

Various code style fixes suggested by @bassosimone
---
 ooni/reporter.py|  6 +++---
 ooni/resources.py   |  4 
 ooni/scripts/ooniprobe.py   | 10 ++
 ooni/scripts/ooniprobe_agent.py | 12 +---
 ooni/scripts/ooniresources.py   |  3 ++-
 5 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/ooni/reporter.py b/ooni/reporter.py
index 951602f..ab32c54 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -221,7 +221,8 @@ class NJSONReporter(OReporter):
 OReporter.__init__(self, test_details)
 
 def _writeln(self, line):
-self._write("%s\n" % line)
+self._write(line)
+self._write("\n")
 
 def _write(self, data):
 if not self._stream:
@@ -249,8 +250,7 @@ class NJSONReporter(OReporter):
 'test_keys': e
 }
 report_entry.update(self.testDetails)
-self._write(json.dumps(report_entry))
-self._write("\n")
+self._writeln(json.dumps(report_entry))
 
 def createReport(self):
 self._stream = open(self.report_path, 'w+')
diff --git a/ooni/resources.py b/ooni/resources.py
index ac0f967..edb7781 100644
--- a/ooni/resources.py
+++ b/ooni/resources.py
@@ -91,6 +91,10 @@ def check_for_update(country_code=None):
 Checks if we need to update the resources.
 If the country_code is specified then only the resources for that
 country will be updated/downloaded.
+
+XXX we currently don't check the shasum of resources although this is
+included inside of the manifest.
+This should probably be done once we have signing of resources.
 :return: the latest version.
 """
 temporary_files = []
diff --git a/ooni/scripts/ooniprobe.py b/ooni/scripts/ooniprobe.py
index d67dd80..c8ffee2 100644
--- a/ooni/scripts/ooniprobe.py
+++ b/ooni/scripts/ooniprobe.py
@@ -12,10 +12,12 @@ def ooniprobe(reactor):
 check_incoherences=True)
 if global_options['queue']:
 return runWithDaemonDirector(global_options)
-elif global_options['initialize']:
+
+if global_options['initialize']:
 initializeOoniprobe(global_options)
 return defer.succeed(None)
-elif global_options['web-ui']:
+
+if global_options['web-ui']:
 from ooni.scripts.ooniprobe_agent import WEB_UI_URL
 from ooni.scripts.ooniprobe_agent import status_agent, start_agent
 if status_agent() != 0:
@@ -25,8 +27,8 @@ def ooniprobe(reactor):
 print("Started ooniprobe-agent")
 webbrowser.open_new(WEB_UI_URL)
 return defer.succeed(None)
-else:
-return runWithDirector(global_options)
+
+return runWithDirector(global_options)
 
 def run():
 task.react(ooniprobe)
diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py
index c455f9b..479d230 100644
--- a/ooni/scripts/ooniprobe_agent.py
+++ b/ooni/scripts/ooniprobe_agent.py
@@ -1,6 +1,7 @@
 from __future__ import print_function
 
 import os
+import sys
 import time
 import errno
 import signal
@@ -65,8 +66,9 @@ def start_agent(options=None):
 twistd_args.append("StartOoniprobeAgent")
 try:
 twistd_config.parseOptions(twistd_args)
-except usage.error, ue:
+except usage.error as ue:
 print("ooniprobe: usage error from twistd: {}\n".format(ue))
+sys.exit(1)
 twistd_config.loadedPlugins = {
 "StartOoniprobeAgent": StartOoniprobeAgentPlugin()
 }
@@ -103,12 +105,8 @@ def get_running_pidfile():
 pid = open(pidfile, "r").read()
 pid = int(pid)
 if is_process_running(pid):
-running_pidfile = pidfile
-else:
-continue
-if running_pidfile is None:
-raise NotRunning
-return running_pidfile
+return pidfile
+raise NotRunning
 
 def status_agent():
 try:
diff --git a/ooni/scripts/ooniresources.py b/ooni/scripts/ooniresources.py
index 5fdbbf0..8e77431 100644
--- a/ooni/scripts/ooniresources.py
+++ b/ooni/scripts/ooniresources.py
@@ -30,5 +30,6 @@ def run():
 print "%s: Try --help for usage details." % (sys.argv[0])
 sys.exit(1)
 
-print("WARNING: Usage of this script is deprecated.")
+print("WARNING: Usage of this script is deprecated. We will not do "
+  "anything.")
 sys.exit(0)



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] First kill ooniprobe-agent with SIGTERM and then hit it harder with SIGKILL

[art]

commit 16fa7de0af33194376ea1da6a8813ffe85d18a81
Author: Arturo Filastò 
Date:   Fri Sep 16 11:57:32 2016 +0200

First kill ooniprobe-agent with SIGTERM and then hit it harder with SIGKILL
---
 ooni/scripts/ooniprobe_agent.py | 20 
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py
index 479d230..4c97f5e 100644
--- a/ooni/scripts/ooniprobe_agent.py
+++ b/ooni/scripts/ooniprobe_agent.py
@@ -128,12 +128,15 @@ def stop_agent():
 pid = open(pidfile, "r").read()
 pid = int(pid)
 try:
-os.kill(pid, signal.SIGKILL)
+os.kill(pid, signal.SIGTERM)
 except OSError as ose:
 if ose.errno == errno.ESRCH:
 print("No process was running. Cleaning up.")
 # the process didn't exist, so wipe the pid file
-os.remove(pidfile)
+try:
+os.remove(pidfile)
+except EnvironmentError as exc:
+print("Failed to delete the pidfile {0}".format(exc))
 return 2
 elif ose.errno == errno.EPERM:
 # The process is owned by root. We assume it's running
@@ -152,7 +155,7 @@ def stop_agent():
 while True:
 # poll once per second until we see the process is no longer running
 try:
-os.kill(pid, 0)
+os.kill(pid, signal.SIG_DFL)
 except OSError:
 print("process %d is dead" % pid)
 return
@@ -161,7 +164,16 @@ def stop_agent():
 if first_time:
 print("It looks like pid %d is still running "
   "after %d seconds" % (pid, (time.time() - start)))
-print("I will keep watching it until you interrupt me.")
+print("Sending a SIGKILL and waiting for it to terminate "
+  "until you kill me.")
+try:
+os.kill(pid, signal.SIGKILL)
+except OSError as ose:
+# Race condition check. It could have dies already. If
+# so we are happy.
+if ose.errno == errno.ESRCH:
+print("process %d is dead" % pid)
+return
 wait = 10
 first_time = False
 else:



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] The version numbers of oonideckgen, oonireport and ooniresources are now the same of ooniprobe

[art]

commit 9f92dc3eefef065ef0be7b577950e2c916f2cc47
Author: Arturo Filastò 
Date:   Fri Sep 16 11:57:02 2016 +0200

The version numbers of oonideckgen, oonireport and ooniresources are now 
the same of ooniprobe
---
 ooni/scripts/oonideckgen.py   | 4 +---
 ooni/scripts/oonireport.py| 3 +--
 ooni/scripts/ooniresources.py | 3 ++-
 3 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/ooni/scripts/oonideckgen.py b/ooni/scripts/oonideckgen.py
index 4ad0f59..ba27ffe 100644
--- a/ooni/scripts/oonideckgen.py
+++ b/ooni/scripts/oonideckgen.py
@@ -11,10 +11,8 @@ from ooni.otime import prettyDateNowUTC
 from ooni import errors
 from ooni.geoip import probe_ip
 from ooni.resources import check_for_update
-from ooni.settings import config
 from ooni.deck import NGDeck
-
-__version__ = "1.0.0"
+from ooni import __version__
 
 class Options(usage.Options):
 synopsis = """%s [options]
diff --git a/ooni/scripts/oonireport.py b/ooni/scripts/oonireport.py
index cd9b244..f7896aa 100644
--- a/ooni/scripts/oonireport.py
+++ b/ooni/scripts/oonireport.py
@@ -14,8 +14,7 @@ from ooni.reporter import OONIBReporter, OONIBReportLog
 from ooni.utils import log
 from ooni.settings import config
 from ooni.backend_client import BouncerClient, CollectorClient
-
-__version__ = "0.1.0"
+from ooni import __version__
 
 @defer.inlineCallbacks
 def lookup_collector_client(report_header, bouncer):
diff --git a/ooni/scripts/ooniresources.py b/ooni/scripts/ooniresources.py
index 8e77431..2ce5c9e 100644
--- a/ooni/scripts/ooniresources.py
+++ b/ooni/scripts/ooniresources.py
@@ -1,6 +1,7 @@
 import sys
 
 from twisted.python import usage
+from ooni import __version__
 
 class Options(usage.Options):
 synopsis = """%s
@@ -17,7 +18,7 @@ class Options(usage.Options):
 optParameters = []
 
 def opt_version(self):
-print("ooniresources version: 0.2.0")
+print("ooniresources version: %s" % __version__)
 sys.exit(0)
 
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Add some debugging logs to the refresh deck list

[art]

commit a3700b5eabd923886f0e6951784196bdc451caaa
Author: Arturo Filastò 
Date:   Tue Sep 13 17:20:34 2016 +0200

Add some debugging logs to the refresh deck list
---
 ooni/agent/scheduler.py | 5 +
 1 file changed, 5 insertions(+)

diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index d6fbc31..2797fba 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -278,6 +278,7 @@ class RefreshDeckList(ScheduledTask):
 def task(self):
 self.scheduler.refresh_deck_list()
 
+
 class SendHeartBeat(ScheduledTask):
 """
 This task is used to send a heartbeat that the probe is still alive and
@@ -345,6 +346,7 @@ class SchedulerService(service.MultiService):
 
 # If we are not initialized we should not enable anything
 if not config.is_initialized():
+log.msg("We are not initialized skipping setup of decks")
 to_enable = []
 
 for scheduled_task in self._scheduled_tasks[:]:
@@ -355,13 +357,16 @@ class SchedulerService(service.MultiService):
 if info in to_enable:
 # If the task is already scheduled there is no need to
 # enable it.
+log.msg("The deck {0} is already scheduled".format(deck_id))
 to_enable.remove(info)
 else:
 # If one of the tasks that is scheduled is no longer in the
 # scheduled tasks. We should disable it.
+log.msg("The deck task {0} should be disabled".format(deck_id))
 self.unschedule(scheduled_task)
 
 for deck_id, schedule in to_enable:
+log.msg("Scheduling to run {0}".format(deck_id))
 self.schedule(RunDeck(self.director, deck_id, schedule))
 
 def _task_did_not_run(self, failure, task):



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Make changes to the updater based on feedback by @bassosimone

[art]

commit 45993baf6bf0ae5377dcc9f99d9bf1c19a050b0b
Author: Arturo Filastò 
Date:   Thu Sep 15 12:41:42 2016 +0200

Make changes to the updater based on feedback by @bassosimone
---
 MANIFEST.in   |   2 +-
 data/lepidopter-update.py | 385 ++
 data/updater.py   | 357 --
 setup.py  |   2 +-
 4 files changed, 387 insertions(+), 359 deletions(-)

diff --git a/MANIFEST.in b/MANIFEST.in
index 0528d4b..258459e 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -7,7 +7,7 @@ include data/oonireport.1
 include data/ooniresources.1
 include data/ooniprobe.conf.sample
 
-include data/updater.py
+include data/lepidopter-update.py
 
 include ooni/settings.ini
 include ooni/ui/consent-form.md
diff --git a/data/lepidopter-update.py b/data/lepidopter-update.py
new file mode 100755
index 000..bdd1cb5
--- /dev/null
+++ b/data/lepidopter-update.py
@@ -0,0 +1,385 @@
+#!/usr/bin/env python2
+"""
+This is the auto-updater script for lepidopter.
+
+It must be run from root and it takes care of downloading the most recent
+updates and doing all the operations needed to perform the update.
+
+To run it expects systemd to be configured.
+
+This script includes a self-installer which can be run via:
+
+python updater.py install
+
+It then expects to be run as a systemd service with:
+
+python updater.py update --watch
+"""
+
+from __future__ import print_function
+
+import os
+import re
+import imp # XPY3 this is deprecated in python3
+import sys
+import time
+import errno
+import shutil
+import getpass
+import logging
+import tempfile
+import argparse
+
+from subprocess import check_output, check_call, CalledProcessError
+
+# The version number of the updater
+__version__ = "1.0.0"
+
+LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"
+# UPDATE_BASE_URL/latest/version must return an integer containing the latest 
version number
+# UPDATE_BASE_URL/VERSION/update.py must return the update script for VERSION
+# UPDATE_BASE_URL/VERSION/update.py.asc must return a valid GPG signature for 
update.py
+UPDATE_BASE_URL = 
"https://github.com/OpenObservatory/lepidopter-update/releases/download/;
+
+CURRENT_VERSION_PATH = "/etc/lepidopter-update/version"
+UPDATER_PATH = "/opt/ooni/lepidopter-update/versions/"
+SCRIPT_INSTALL_PATH = "/opt/ooni/lepidopter-update/updater.py"
+
+SYSTEMD_SCRIPT_PATH = "/etc/systemd/system/lepidopter-update.service"
+SYSTEMD_SCRIPT = """\
+[Unit]
+Description=lepidopter-update service
+
+[Service]
+Type=simple
+ExecStart={0} --log-file /var/log/ooni/lepidopter-update.log update --watch
+TimeoutStartSec=300
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
+""".format(SCRIPT_INSTALL_PATH)
+
+PUBLIC_KEY_PATH = "/opt/ooni/lepidopter-update/public.asc"
+PUBLIC_KEY = """\
+-BEGIN PGP PUBLIC KEY BLOCK-
+Comment: GPGTools - https://gpgtools.org
+
+mQINBFfEAKABEADNBPp2nD48xXRhMdKMVXS2qHgDzokSAn3hikA+cb2IL5ssde0o
+9HHzMxSNCbQBWo1bpmg84zsHvZTL+yEVGJ+o8DjLfdKKdMUOPsLTc0O1rqD0M6L4
+35n6JjaeJp98HhVIRkmNqBG4pWMKLqvW1crEt5U8m/X7LWtTzsBt2DPi6UB6yDqw
+520DLK051/0WKE+s7W8f8hYheHqyaUl35wtU6Qj7kjcDm0Kg57l7pY7gdYEeRizA
+TECXy2c2mKJusql3p65FD/jNX6TncfHWiESvS8p31E8xx1hfgsgmh15JqrMTALm/
+7cn3/IDV5vPBzi2pf4IlVHo34QcE26uj7QaXjrlQUkuds5cAFy/4uozN6J2PbH2x
+e1+oI9rGxSf9m7UfAbudC+QATAlMDNeH2ngeqA0tm4vrMk/ybj5efeUjGNGNW0c8
+6xfhbyhNJb6Rw2ScwdFUc/niWone3O1J3QkQ6CS6/gT3JCBMRVwLl+CkbeaALBTI
+6We0CNQc1FXcWB84LI9F3UAHiR9jrmA3J/ck4R1oqv9STTrClTdWIvCK4sNa0sv7
+ra1fdEV4CK1Z0qKxbKCk/JTlD/9w/OqZQqyJLOrWXomYxR6I6lxNwhoC+3Ysj5EG
+Mmagpi+nnqAK0oIBkPytts9e6e1D54hS9sEG4uaEQRm229e0yhmQNQOKNwARAQAB
+tDZPT05JIHNvZnR3YXJlIHVwZGF0ZSBrZXkgPGNvbnRhY3RAb3Blbm9ic2VydmF0
+b3J5Lm9yZz6JAj0EEwEKACcFAlfEAKACGwMFCQHhM4AFCwkIBwMFFQoJCAsFFgID
+AQACHgECF4AACgkQw+zcBCBPnSm6ug//eVOV7RiG8q5ry64TvgeTNfPlVF0R3y3d
+2dUNaWy+4H0ay9UjW/ayxZNnSSreVZY+50pOiqsKWdV5bEgtOZXkDfth8NuCNddo
+CYmVkV/x2Mvmpf3eTBXlXtmFn9j2an3GKSSHFscdfdZsPATUUv+YFyX8LK5K6vq+
+BdNEGpqqHxPEM0wyQm2/f2s0dmjkmPFNZpCGWnuBRpQQD5O2YwFKK316VNdBXvVA
+i9+MA81vLtn40FsOKZ/kDLt65khEdgYTYj8lRXIEWGuWp1iPUuMmEL8dxtlY8K1R
+qU2JbgHHOA7RHnAUqgg0Hjmyg4ZsQ/ZyWi2/3IoLn/7QGeV0HBdiGMFuShSfkFWx
+bNNMuei9FVK4nwXRLcVfAMXv1GtqQU9jTeCYXzxgr81rkEivkdqlZ3Iins+KgWEQ
+SbEEYAXOWp/oheTBOBQvLSZi+2vjMiUeIQHQUDNfhlp3/Mk6RTVLMml6thIY/NyL
+f/vABO5V9oKAdIaFMu/70tYn8PxTqPE0uJ7FwcTa7awp10dkpXXk0tm5ywYsms8l
+CA/vizq7VMiZC9G4JvZqa3vXNBT1yFe+4Ri+fLtdZw9IDgECi5ZdQlp7dx2Rei2i
+S2XkUwWR4Qv3/WzvPDChr25BMlu0Pkb8MbynrxcMs5ODFxOuOiP2kL4YW2Qppo66
+U3Z92swhAIq5Ag0EV8QAoAEQAOQwsRo+2260kBYKnxRHr6rzTjStXtxsCsMUB08E
+XS7eTElwDSE2C+pfeQjFe366f1zNTxY/CN6wCtd7wI4cVXWKLescFfCUrsg+S0Wf
+ot85AXqCqrPKFtKwW8khUeVnQfmHwhQl1W+/t+bE2p4X+0OR8qugHsMnvYwl+KpK
+sZ094LwkO8GRySB+LKm6KQtJ+WOnsvs3X8v8fSA6GwJjYdtKqNUzPBLpw8RrIH9l
+eaT2pe9Ta48GqEwrU8wxwKyRBIfJJP/zq5n1rKcOBpvLZDVcyrVw+pIGa0zfmr/c
+qWYG7znx2Xq3i22d36xPkfkZEyVnQcCJJ28hkAfXRYpp+gMnL0Zt4u3GgzSARSBS
+VrcMyNlaft/aSOkojyjh3+2zF1PCfW1Nw9Sx50gdN3FfF0yEWjUoA1R/NW9CQZVG

[tor-commits] [ooni-probe/master] Force creation of decks-available directory.

[art]

commit 28a441b1da39b8b8de28752dcfc0578352f6911c
Author: Arturo Filastò 
Date:   Tue Sep 13 15:28:02 2016 +0200

Force creation of decks-available directory.

Failing to do so will lead to the decks not being copied over and pip will
silently fail.
---
 setup.py | 19 +++
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/setup.py b/setup.py
index 60d08a5..753cf07 100644
--- a/setup.py
+++ b/setup.py
@@ -87,6 +87,7 @@ Have fun!
 from __future__ import print_function
 
 import os
+import errno
 import tempfile
 from glob import glob
 
@@ -147,6 +148,13 @@ def install_lepidopter_update():
 check_call(["data/updater.py", "install"])
 
 
+def mkdir_p(path):
+try:
+os.makedirs(path)
+except OSError as ose:
+if ose != errno.EEXIST:
+raise
+
 class OoniInstall(InstallCommand):
 def gen_config(self, share_path):
 config_file = pj(tempfile.mkdtemp(), "ooniprobe.conf.sample")
@@ -183,14 +191,9 @@ class OoniInstall(InstallCommand):
 with open("ooni/settings.ini", "w+") as fp:
 settings.write(fp)
 
-try:
-os.makedirs(pj(var_path, 'ooni'))
-except OSError:
-pass
-try:
-os.makedirs(pj(share_path, 'ooni'))
-except OSError:
-pass
+mkdir_p(pj(var_path, 'ooni'))
+mkdir_p(pj(share_path, 'ooni'))
+mkdir_p(pj(share_path, 'ooni', 'decks-available'))
 
 def pre_install(self):
 prefix = os.path.abspath(self.prefix)



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] The last runtime should be determined based on the time we started the task

[art]

commit 5c0fd074630b7fdd4e066c0b783242880d0838a3
Author: Arturo Filastò 
Date:   Mon Sep 12 14:53:32 2016 +0200

The last runtime should be determined based on the time we started the task
---
 ooni/agent/scheduler.py | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 4002369..a7713e0 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -93,10 +93,9 @@ class ScheduledTask(object):
 date_str = in_file.read()
 return datetime.strptime(date_str, self._time_format)
 
-def _update_last_run(self):
+def _update_last_run(self, last_run_time):
 with self._last_run.open('w') as out_file:
-current_time = datetime.utcnow()
-out_file.write(current_time.strftime(self._time_format))
+out_file.write(last_run_time.strftime(self._time_format))
 
 def task(self):
 raise NotImplementedError
@@ -117,8 +116,9 @@ class ScheduledTask(object):
 try:
 if self.last_run == CANARY_DATE:
 yield defer.maybeDeferred(self.first_run)
+last_run_time = datetime.utcnow()
 yield self.task()
-self._update_last_run()
+self._update_last_run(last_run_time)
 except:
 raise
 finally:



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] ooniprobe_agent: fix behavior when no command is specified

[art]

commit 300660ff9873d45d78e54b4a9c09c7c659afe6f5
Author: Simone Basso 
Date:   Mon Sep 19 11:02:17 2016 +0200

ooniprobe_agent: fix behavior when no command is specified
---
 ooni/scripts/ooniprobe_agent.py | 21 +++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py
index 4c97f5e..49694eb 100644
--- a/ooni/scripts/ooniprobe_agent.py
+++ b/ooni/scripts/ooniprobe_agent.py
@@ -12,6 +12,7 @@ from twisted.python import usage
 from ooni.utils import log, is_process_running
 from ooni.settings import config
 from ooni.agent.agent import AgentService
+from ooni import __version__
 
 
 class StartOoniprobeAgentPlugin:
@@ -38,6 +39,10 @@ class RunOptions(usage.Options):
 pass
 
 class AgentOptions(usage.Options):
+
+synopsis = """%s [options] command
+""" % (os.path.basename(sys.argv[0]),)
+
 subCommands = [
 ['start', None, StartOptions, "Start the ooniprobe-agent in the "
   "background"],
@@ -45,9 +50,17 @@ class AgentOptions(usage.Options):
 ['status', None, StatusOptions, "Show status of the ooniprobe-agent"],
 ['run', None, RunOptions, "Run the ooniprobe-agent in the foreground"]
 ]
+
 def postOptions(self):
 self.twistd_args = []
 
+def opt_version(self):
+"""
+Display the ooniprobe version and exit.
+"""
+print("ooniprobe-agent version:", __version__)
+sys.exit(0)
+
 def start_agent(options=None):
 config.set_paths()
 config.initialize_ooni_home()
@@ -188,8 +201,9 @@ def run():
 options = AgentOptions()
 options.parseOptions()
 
-if options.subCommand == "run":
-options.twistd_args += ("--nodaemon",)
+if options.subCommand == None:
+print(options)
+return
 
 if options.subCommand == "stop":
 return stop_agent()
@@ -197,6 +211,9 @@ def run():
 if options.subCommand == "status":
 return status_agent()
 
+if options.subCommand == "run":
+options.twistd_args += ("--nodaemon",)
+
 return start_agent(options)
 
 if __name__ == "__main__":



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Bump to release candidate 3

[art]

commit b6b03acf87c44e798480deefd86ce2bb5926b7c0
Author: Arturo Filastò 
Date:   Mon Sep 19 13:38:27 2016 +0200

Bump to release candidate 3
---
 ooni/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ooni/__init__.py b/ooni/__init__.py
index 20bdef0..41f5c37 100644
--- a/ooni/__init__.py
+++ b/ooni/__init__.py
@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 
 __author__ = "Open Observatory of Network Interference"
-__version__ = "2.0.0-rc.2"
+__version__ = "2.0.0-rc.3"
 
 __all__ = [
 'agent',



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Move check for running process into utility function

[art]

commit 8c3a65edc75e6830f4e5c7a6c2e1afd55851de3e
Author: Arturo Filastò 
Date:   Thu Sep 15 16:22:12 2016 +0200

Move check for running process into utility function

* More fixes based on review by @bassosimone
---
 ooni/measurements.py| 18 +++---
 ooni/nettest.py |  2 +-
 ooni/scripts/ooniprobe_agent.py | 16 
 ooni/utils/__init__.py  | 15 +++
 4 files changed, 23 insertions(+), 28 deletions(-)

diff --git a/ooni/measurements.py b/ooni/measurements.py
index 294b182..422fc55 100644
--- a/ooni/measurements.py
+++ b/ooni/measurements.py
@@ -1,9 +1,7 @@
-import os
 import json
-import signal
 
 from twisted.python.filepath import FilePath
-from ooni.utils import log
+from ooni.utils import log, is_process_running
 from ooni.utils.files import directory_usage
 from ooni.settings import config
 
@@ -75,14 +73,11 @@ def get_measurement(measurement_id, compute_size=False):
 stale = False
 if measurement.child("measurements.njson.progress").exists():
 completed = False
-# XXX this is done quite often around the code, probably should
-# be moved into some utility function.
 pid = measurement.child("running.pid").open("r").read()
 pid = int(pid)
-try:
-os.kill(pid, signal.SIG_DFL)
+if is_process_running(pid):
 running = True
-except OSError:
+else:
 stale = True
 
 if measurement.child("keep").exists():
@@ -136,10 +131,3 @@ def list_measurements(compute_size=False):
 except:
 log.err("Failed to get metadata for measurement 
{0}".format(measurement_id))
 return measurements
-
-if __name__ == "__main__":
-import sys
-if len(sys.argv) != 3:
-print("Usage: {0} [input_file] [output_file]".format(sys.argv[0]))
-sys.exit(1)
-generate_summary(sys.argv[1], sys.argv[2])
diff --git a/ooni/nettest.py b/ooni/nettest.py
index 4ea3329..d6784f4 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -242,7 +242,7 @@ class NetTestLoader(object):
 m = ONION_INPUT_REGEXP.match(filename)
 if m:
 raise e.InvalidInputFile("Input files hosted on hidden services "
- "are not longer supported")
+ "are no longer supported")
 else:
 input_file['filename'] = filename
 self.inputFiles.append(input_file)
diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py
index 3eb1d22..c455f9b 100644
--- a/ooni/scripts/ooniprobe_agent.py
+++ b/ooni/scripts/ooniprobe_agent.py
@@ -8,7 +8,7 @@ import signal
 from twisted.scripts import twistd
 from twisted.python import usage
 
-from ooni.utils import log
+from ooni.utils import log, is_process_running
 from ooni.settings import config
 from ooni.agent.agent import AgentService
 
@@ -102,18 +102,10 @@ def get_running_pidfile():
 continue
 pid = open(pidfile, "r").read()
 pid = int(pid)
-try:
-os.kill(pid, signal.SIG_DFL)
+if is_process_running(pid):
 running_pidfile = pidfile
-break
-except OSError as ose:
-if ose.errno == errno.ESRCH:
-# Found pid, but isn't running
-continue
-elif ose.errno == errno.EPERM:
-# The process is owned by root. We assume it's running
-running_pidfile = pidfile
-break
+else:
+continue
 if running_pidfile is None:
 raise NotRunning
 return running_pidfile
diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py
index 247758f..a894daf 100644
--- a/ooni/utils/__init__.py
+++ b/ooni/utils/__init__.py
@@ -1,6 +1,8 @@
 import shutil
 import string
 import random
+import signal
+import errno
 import gzip
 import os
 
@@ -160,3 +162,16 @@ def gunzip(file_path):
 def get_ooni_root():
 script = os.path.join(__file__, '..')
 return os.path.dirname(os.path.realpath(script))
+
+def is_process_running(pid):
+try:
+os.kill(pid, signal.SIG_DFL)
+running = True
+except OSError as ose:
+if ose.errno == errno.EPERM:
+running = True
+elif ose.errno == errno.ESRCH:
+running = False
+else:
+raise
+return running



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] In virtualenvs the /usr/share path is actually prefix/share/

[art]

commit 38775a3d39f598e925170f5f30792a44415a8a74
Author: Arturo Filastò 
Date:   Tue Sep 13 15:41:11 2016 +0200

In virtualenvs the /usr/share path is actually prefix/share/
---
 ooni/settings.py | 2 +-
 setup.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/ooni/settings.py b/ooni/settings.py
index 409e81e..be0025d 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -168,7 +168,7 @@ if IS_VIRTUALENV:
 )
 USR_SHARE_PATH = os.path.join(
 _PREFIX,
-'usr', 'share', 'ooni'
+'share', 'ooni'
 )
 ETC_PATH = os.path.join(
 _PREFIX,
diff --git a/setup.py b/setup.py
index 753cf07..e6031c6 100644
--- a/setup.py
+++ b/setup.py
@@ -152,7 +152,7 @@ def mkdir_p(path):
 try:
 os.makedirs(path)
 except OSError as ose:
-if ose != errno.EEXIST:
+if ose.errno != errno.EEXIST:
 raise
 
 class OoniInstall(InstallCommand):



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Bump the timeout value for test_oonicli.

[art]

commit 4fd30aab622dcd9dc50311dd8e1ccf58cdafaa87
Author: Arturo Filastò 
Date:   Fri Sep 16 18:23:40 2016 +0200

Bump the timeout value for test_oonicli.

Travis has gotten slower or our tests have gotten slower, either way let's
increase this to avoid all the annoying transient travis failures.
---
 ooni/tests/test_oonicli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py
index 8ad8a36..4a6edf9 100644
--- a/ooni/tests/test_oonicli.py
+++ b/ooni/tests/test_oonicli.py
@@ -59,7 +59,7 @@ tor:
 
 
 class TestRunDirector(ConfigTestCase):
-timeout = 220
+timeout = 420
 
 def setUp(self):
 super(TestRunDirector, self).setUp()



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Refactor the pattern for directory creation into a function

[art]

commit ba8558514aabc963782688ac694daa196c53dc8d
Author: Arturo Filastò 
Date:   Fri Sep 16 12:10:20 2016 +0200

Refactor the pattern for directory creation into a function
---
 ooni/deck/store.py  | 14 --
 ooni/resources.py   | 16 +---
 ooni/scripts/oonideckgen.py |  8 ++--
 ooni/utils/__init__.py  | 13 +
 ooni/utils/log.py   |  9 ++---
 ooni/utils/onion.py |  8 +++-
 6 files changed, 29 insertions(+), 39 deletions(-)

diff --git a/ooni/deck/store.py b/ooni/deck/store.py
index bd852c7..bf1eb7f 100644
--- a/ooni/deck/store.py
+++ b/ooni/deck/store.py
@@ -6,6 +6,7 @@ from copy import deepcopy
 from twisted.internet import defer
 from twisted.python.filepath import FilePath
 
+from ooni.utils import mkdir_p
 from ooni.deck.deck import NGDeck
 from ooni.otime import timestampNowISO8601UTC
 from ooni.resources import check_for_update
@@ -80,16 +81,9 @@ class InputStore(object):
 self.path = FilePath(config.inputs_directory)
 self.resources = FilePath(config.resources_directory)
 
-try:
-self.path.child("descriptors").makedirs()
-except OSError as e:
-if not e.errno == errno.EEXIST:
-raise
-try:
-self.path.child("data").makedirs()
-except OSError as e:
-if not e.errno == errno.EEXIST:
-raise
+mkdir_p(self.path.child("descriptors").path)
+mkdir_p(self.path.child("data").path)
+
 yield self.update_url_lists(country_code)
 
 @defer.inlineCallbacks
diff --git a/ooni/resources.py b/ooni/resources.py
index edb7781..1830d42 100644
--- a/ooni/resources.py
+++ b/ooni/resources.py
@@ -5,7 +5,7 @@ from twisted.python.filepath import FilePath
 from twisted.internet import defer
 from twisted.web.client import downloadPage, getPage, HTTPClientFactory
 
-from ooni.utils import log, gunzip, rename
+from ooni.utils import log, gunzip, rename, mkdir_p
 from ooni.settings import config
 
 # Disable logs of HTTPClientFactory
@@ -107,11 +107,7 @@ def check_for_update(country_code=None):
 latest_version = yield get_latest_version()
 
 resources_dir = FilePath(config.resources_directory)
-try:
-resources_dir.makedirs()
-except OSError as e:
-if not e.errno == errno.EEXIST:
-raise
+mkdir_p(resources_dir.path)
 current_manifest = resources_dir.child("manifest.json")
 
 if current_manifest.exists():
@@ -153,11 +149,9 @@ def check_for_update(country_code=None):
 filename = filename[:-3]
 gzipped = True
 dst_file = resources_dir.child(pre_path).child(filename)
-try:
-dst_file.parent().makedirs()
-except OSError as e:
-if not e.errno == errno.EEXIST:
-raise
+
+mkdir_p(dst_file.parent().path)
+
 src_file = dst_file.temporarySibling()
 src_file.alwaysCreate = 0
 
diff --git a/ooni/scripts/oonideckgen.py b/ooni/scripts/oonideckgen.py
index ba27ffe..89541d7 100644
--- a/ooni/scripts/oonideckgen.py
+++ b/ooni/scripts/oonideckgen.py
@@ -1,12 +1,12 @@
 from __future__ import print_function
 
-import errno
 import os
 import sys
 
 from twisted.internet import defer, task
 from twisted.python import usage
 
+from ooni.utils import mkdir_p
 from ooni.otime import prettyDateNowUTC
 from ooni import errors
 from ooni.geoip import probe_ip
@@ -120,11 +120,7 @@ def oonideckgen(reactor):
 
 options['country-code'] = options['country-code'].lower()
 
-try:
-os.makedirs(os.path.dirname(options['output']))
-except OSError as exception:
-if exception.errno != errno.EEXIST:
-raise
+mkdir_p(os.path.dirname(options['output']))
 
 generate_deck(options)
 
diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py
index a894daf..99f1985 100644
--- a/ooni/utils/__init__.py
+++ b/ooni/utils/__init__.py
@@ -175,3 +175,16 @@ def is_process_running(pid):
 else:
 raise
 return running
+
+def mkdir_p(path):
+"""
+Like makedirs, but it also ignores EEXIST errors, unless it exists but
+isn't a directory.
+"""
+try:
+os.makedirs(path)
+except OSError as ose:
+if ose.errno != errno.EEXIST:
+raise
+if not os.path.isdir(path):
+raise
diff --git a/ooni/utils/log.py b/ooni/utils/log.py
index 93e6b31..f20fdce 100644
--- a/ooni/utils/log.py
+++ b/ooni/utils/log.py
@@ -7,6 +7,7 @@ import logging
 from twisted.python import log as tw_log
 from twisted.python.logfile import DailyLogFile
 
+from ooni.utils import mkdir_p
 from ooni import otime
 
 # Get rid of the annoying "No route found for
@@ -140,13 +141,7 @@ class OONILogger(object):
 log_folder = config.running_path
 logfile = os.path.join(log_folder, "ooniprobe.log")
 
-try:
-   

[tor-commits] [ooni-probe/master] Fix cache updating logic.

[art]

commit 79ed2a50afc489b925bd6eaa78f1342d60877955
Author: Arturo Filastò 
Date:   Fri Sep 16 17:52:06 2016 +0200

Fix cache updating logic.

* Fix typo in comment
---
 ooni/deck/store.py| 11 ---
 ooni/ui/web/server.py |  4 ++--
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/ooni/deck/store.py b/ooni/deck/store.py
index bf1eb7f..f30b0d6 100644
--- a/ooni/deck/store.py
+++ b/ooni/deck/store.py
@@ -92,15 +92,17 @@ class InputStore(object):
 yield self.create(country_code)
 
 def _update_cache(self):
+new_cache = {}
 descs = self.path.child("descriptors")
 if not descs.exists():
-self._cache = {}
+self._cache = new_cache
 return
 
 for fn in descs.listdir():
 with descs.child(fn).open("r") as in_fh:
 input_desc = json.load(in_fh)
-self._cache[input_desc.pop("id")] = input_desc
+new_cache[input_desc.pop("id")] = input_desc
+self._cache = new_cache
 self._cache_stale = False
 return
 
@@ -172,6 +174,7 @@ class DeckStore(object):
 deck_enabled_path.remove()
 
 def _update_cache(self):
+new_cache = {}
 for deck_path in self.available_directory.listdir():
 if not deck_path.endswith('.yaml'):
 continue
@@ -179,7 +182,9 @@ class DeckStore(object):
 deck = NGDeck(
 deck_path=self.available_directory.child(deck_path).path
 )
-self._cache[deck_id] = deck
+new_cache[deck_id] = deck
+self._cache = new_cache
+self._cache_stale = False
 
 def get(self, deck_id):
 if self._cache_stale:
diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index 48f21c7..cee42ce 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -144,8 +144,8 @@ class LongPoller(object):
 
 class WebUIAPI(object):
 app = Klein()
-# Maximum number in seconds after which to return a result even if not
-# change happenned.
+# Maximum number in seconds after which to return a result even if no
+# change happened.
 _long_polling_timeout = 5
 _reactor = reactor
 _enable_xsrf_protection = True



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Revert "ui/web/server.py: pass binary, not unicode cookies"

[art]

commit baee35fe51e70547e6128885ee0a97e1937d95d0
Author: Simone Basso 
Date:   Mon Sep 19 11:37:20 2016 +0200

Revert "ui/web/server.py: pass binary, not unicode cookies"

This reverts commit 21548e9f0f53100336ef4f22d423f5f728a117e5 since
now Travis is failing for all versions of Twisted.
---
 ooni/ui/web/server.py | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index 385bf18..eec82d3 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -56,13 +56,13 @@ def xsrf_protect(check=True):
 @wraps(f)
 def wrapper(instance, request, *a, **kw):
 should_check = check and instance._enable_xsrf_protection
-token_cookie = request.getCookie(b'XSRF-TOKEN')
+token_cookie = request.getCookie(u'XSRF-TOKEN')
 token_header = request.getHeader(b"X-XSRF-TOKEN")
 if (token_cookie != instance._xsrf_token and
 instance._enable_xsrf_protection):
-request.addCookie(b'XSRF-TOKEN',
+request.addCookie(u'XSRF-TOKEN',
   instance._xsrf_token,
-  path=b'/')
+  path=u'/')
 if should_check and token_cookie != token_header:
 raise WebUIError(404, "Invalid XSRF token")
 return f(instance, request, *a, **kw)
@@ -161,7 +161,7 @@ class WebUIAPI(object):
 # We use a double submit token to protect against XSRF
 rng = SystemRandom()
 token_space = string.letters+string.digits
-self._xsrf_token = b''.join([rng.choice(token_space)
+self._xsrf_token = ''.join([rng.choice(token_space)
 for _ in range(30)])
 
 self._director_started = False



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Bump up the default timeout for tor to 400 seconds

[art]

commit 3f25dad2ed4ede50798465eb284dbbc8f52e5ee4
Author: Arturo Filastò 
Date:   Tue Sep 13 17:42:33 2016 +0200

Bump up the default timeout for tor to 400 seconds
---
 ooni/settings.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ooni/settings.py b/ooni/settings.py
index be0025d..14f135f 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -86,7 +86,7 @@ tor:
 #
 # This is the timeout after which we consider to to not have
 # bootstrapped properly.
-#timeout: 200
+#timeout: 400
 #torrc:
 #HTTPProxy: host:port
 #HTTPProxyAuthenticator: user:password
@@ -141,7 +141,7 @@ defaults = {
 "control_port": None,
 "bridges": None,
 "data_dir": None,
-"timeout": 200,
+"timeout": 400,
 "torrc": {}
 }
 }



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Merge pull request #582 from TheTorProject/v2.0.0-alpha

[art]

commit 7d5f430becd2372d71c8e5839d32244121dbbe80
Merge: da00195 b6b03ac
Author: Simone Basso 
Date:   Mon Sep 19 13:56:08 2016 +0200

Merge pull request #582 from TheTorProject/v2.0.0-alpha

V2.0.0 alpha

 .coveragerc| 2 +-
 .gitignore | 4 +
 ChangeLog.rst  |18 +
 MANIFEST.in| 8 +-
 Vagrantfile|33 +-
 bin/Makefile   |81 -
 bin/oonid  | 7 -
 bin/oonideckgen|37 -
 bin/ooniprobe  |24 -
 bin/ooniprobe-dev  |10 -
 bin/oonireport |38 -
 bin/ooniresources  |35 -
 bin/test/ooni/__init__.py  | 1 -
 data/configs/lepidopter-ooniprobe.conf |75 -
 data/configs/lepidopter-oonireport.conf|69 -
 data/decks/complete.deck   |72 -
 data/decks/complete_no_root.deck   |60 -
 data/decks/fast.deck   |36 -
 data/decks/fast_no_root.deck   |24 -
 data/decks/mlab.deck   |12 -
 data/decks/web-full.yaml   |23 +
 data/decks/web-no-invalid.yaml |18 +
 data/inputs/.gitignore | 2 -
 data/lepidopter-update.py  |   385 +
 data/ooniprobe.conf.sample | 7 +-
 data/resources/README  | 2 -
 data/ui/.bowerrc   | 5 -
 data/ui/app/index.html |39 -
 data/ui/app/libs/angular-resource/README.md| 4 -
 .../app/libs/angular-resource/angular-resource.js  |   445 -
 data/ui/app/libs/angular-resource/component.json   |17 -
 data/ui/app/libs/angular/angular.js| 14733 ---
 data/ui/app/libs/angular/component.json|14 -
 data/ui/app/libs/bootstrap/component.json  | 9 -
 .../libs/bootstrap/css/bootstrap-responsive.css|  1109 --
 data/ui/app/libs/bootstrap/css/bootstrap.css   |  6158 
 .../bootstrap/img/glyphicons-halflings-white.png   |   Bin 8777 -> 0 bytes
 .../libs/bootstrap/img/glyphicons-halflings.png|   Bin 12799 -> 0 bytes
 data/ui/app/libs/bootstrap/js/bootstrap.js |  2276 ---
 data/ui/app/libs/jquery/component.json |14 -
 data/ui/app/libs/jquery/composer.json  |23 -
 data/ui/app/libs/jquery/jquery.js  |  9472 
 data/ui/app/libs/ng-upload/component.json  |23 -
 data/ui/app/libs/ng-upload/ng-upload.js|   107 -
 data/ui/app/scripts/app.js |30 -
 data/ui/app/scripts/controllers.js |96 -
 data/ui/app/scripts/directives.js  | 5 -
 data/ui/app/scripts/filters.js | 5 -
 data/ui/app/scripts/services.js|27 -
 data/ui/app/styles/app.css |21 -
 data/ui/app/views/inputs.html  |31 -
 data/ui/app/views/settings.html| 0
 data/ui/app/views/sidebar.html |10 -
 data/ui/app/views/test.html|48 -
 data/ui/component.json | 9 -
 ooni/__init__.py   |13 +-
 ooni/agent/__init__.py | 0
 ooni/agent/agent.py|28 +
 ooni/agent/scheduler.py|   432 +
 ooni/api/__init__.py   | 0
 ooni/api/spec.py   |   253 -
 ooni/backend_client.py |   117 +-
 ooni/constants.py  | 4 +-
 ooni/contrib/__init__.py   | 1 +
 ooni/contrib/croniter.py   |   430 +
 ooni/contrib/dateutil/__init__.py  | 0
 ooni/contrib/dateutil/relativedelta.py |   539 +
 ooni/contrib/dateutil/tz/__init__.py   | 4 +
 ooni/contrib/dateutil/tz/_common.py|   100 +
 ooni/contrib/dateutil/tz/tz.py |  1339 ++
 ooni/contrib/dateutil/tz/win.py|   354 +
 ooni/deck.py   |   413 -
 ooni/deck/__init__.py  | 1 +
 ooni/deck/backend.py   |   191 +
 ooni/deck/deck.py  |   411 +
 ooni/deck/legacy.py  

[tor-commits] [ooni-probe/master] Skip tests for http_requests integration tests

[art]

commit a89d9124caaac15dd54af44975e6029821c41f78
Author: Arturo Filastò 
Date:   Mon Sep 19 12:43:55 2016 +0200

Skip tests for http_requests integration tests
---
 ooni/tests/test_oonicli.py | 14 +-
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py
index 05bc273..896a4b0 100644
--- a/ooni/tests/test_oonicli.py
+++ b/ooni/tests/test_oonicli.py
@@ -111,7 +111,7 @@ class TestRunDirector(ConfigTestCase):
 
 @defer.inlineCallbacks
 def test_http_requests(self):
-retries = 3
+self.skipTest("XXX This integration test fails non deterministically")
 def verify_function(entry):
 assert 'body_length_match' in entry
 assert 'body_proportion' in entry
@@ -120,17 +120,13 @@ class TestRunDirector(ConfigTestCase):
 assert 'factor' in entry
 assert 'headers_diff' in entry
 assert 'headers_match' in entry
-while retries > 0:
-try:
-yield self.run_helper('blocking/http_requests',
-['-u', 'http://torproject.org/'],
-verify_function)
-break
-except:
-retries -= 1
+yield self.run_helper('blocking/http_requests',
+['-u', 'http://torproject.org/'],
+verify_function)
 
 @defer.inlineCallbacks
 def test_http_requests_with_file(self):
+self.skipTest("XXX This integration test fails non deterministically")
 def verify_function(entry):
 assert 'body_length_match' in entry
 assert 'body_proportion' in entry



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Revert "Revert "ui/web/server.py: pass binary, not unicode cookies""

[art]

commit 897041c08eb6a54e8b181781718412d1500e9800
Author: Simone Basso 
Date:   Mon Sep 19 12:12:42 2016 +0200

Revert "Revert "ui/web/server.py: pass binary, not unicode cookies""

This reverts commit baee35fe51e70547e6128885ee0a97e1937d95d0 because
apparently the error was on a unrelated code path.
---
 ooni/ui/web/server.py | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index eec82d3..385bf18 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -56,13 +56,13 @@ def xsrf_protect(check=True):
 @wraps(f)
 def wrapper(instance, request, *a, **kw):
 should_check = check and instance._enable_xsrf_protection
-token_cookie = request.getCookie(u'XSRF-TOKEN')
+token_cookie = request.getCookie(b'XSRF-TOKEN')
 token_header = request.getHeader(b"X-XSRF-TOKEN")
 if (token_cookie != instance._xsrf_token and
 instance._enable_xsrf_protection):
-request.addCookie(u'XSRF-TOKEN',
+request.addCookie(b'XSRF-TOKEN',
   instance._xsrf_token,
-  path=u'/')
+  path=b'/')
 if should_check and token_cookie != token_header:
 raise WebUIError(404, "Invalid XSRF token")
 return f(instance, request, *a, **kw)
@@ -161,7 +161,7 @@ class WebUIAPI(object):
 # We use a double submit token to protect against XSRF
 rng = SystemRandom()
 token_space = string.letters+string.digits
-self._xsrf_token = ''.join([rng.choice(token_space)
+self._xsrf_token = b''.join([rng.choice(token_space)
 for _ in range(30)])
 
 self._director_started = False



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits