[tor-commits] [torbirdy/master] Replace 'var' with 'let' for iterations

[sukhbir]

commit de8d9fa2964132f657f03d3ba27f3ea317c39eba
Author: Sukhbir Singh 
Date:   Mon May 30 06:19:48 2016 -0400

Replace 'var' with 'let' for iterations
---
 chrome/content/preferences.js | 10 +-
 components/torbirdy.js| 26 +-
 2 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/chrome/content/preferences.js b/chrome/content/preferences.js
index 682b367..c843d3c 100644
--- a/chrome/content/preferences.js
+++ b/chrome/content/preferences.js
@@ -77,7 +77,7 @@ if (!org.torbirdy.prefs) org.torbirdy.prefs = new function() {
 
   pub.clearCustomPrefs = function() {
 var customPrefs = pub.torBirdyPref.getChildList("", {});
-for (var i = 0; i < customPrefs.length; i++) {
+for (let i = 0; i < customPrefs.length; i++) {
   pub.prefs.clearUserPref(pub.customBranch + customPrefs[i]);
 }
   };
@@ -131,7 +131,7 @@ if (!org.torbirdy.prefs) org.torbirdy.prefs = new 
function() {
 
 var accountLength = accounts.length;
 
-for (var i = 0; i < accountLength; i++) {
+for (let i = 0; i < accountLength; i++) {
   var account = accounts.queryElementAt(i, 
Components.interfaces.nsIMsgAccount).incomingServer;
 
   var name = account.prettyName;
@@ -149,7 +149,7 @@ if (!org.torbirdy.prefs) org.torbirdy.prefs = new 
function() {
   // For email accounts, configure accordingly.
   var sAccount = null;
   var account = pub.getAccount();
-  for (var i = 0; i < account.length; i++) {
+  for (let i = 0; i < account.length; i++) {
 if (account[i].key === pub.mailAccount.value) {
   sAccount = i;
 }
@@ -324,14 +324,14 @@ if (!org.torbirdy.prefs) org.torbirdy.prefs = new 
function() {
   var accounts = pub.getAccount();
   if (pub.fetchAllMails.checked) {
 pub.prefs.setBoolPref(pub.prefBranch + 'fetchall', true);
-for (var i = 0; i < accounts.length; i++) {
+for (let i = 0; i < accounts.length; i++) {
   accounts[i].loginAtStartUp = true;
   accounts[i].doBiff = true;
 }
   }
   else {
 pub.prefs.setBoolPref(pub.prefBranch + 'fetchall', false);
-for (var i = 0; i < accounts.length; i++) {
+for (let i = 0; i < accounts.length; i++) {
   accounts[i].loginAtStartUp = false;
   accounts[i].doBiff = false;
 }
diff --git a/components/torbirdy.js b/components/torbirdy.js
index 9c930e8..fa1aa6b 100644
--- a/components/torbirdy.js
+++ b/components/torbirdy.js
@@ -480,13 +480,13 @@ TorBirdy.prototype = {
   resetUserPrefs: function() {
 dump("Resetting user preferences to default\n");
 // Clear the Thunderbird preferences we changed.
-for (var each in TorBirdyPrefs) {
+for (let each in TorBirdyPrefs) {
   this.prefs.clearUserPref(each);
 }
 
 // Restore the older proxy preferences that were set prior to TorBirdy's 
install.
 dump("Restoring proxy settings\n");
-for (var i = 0; i < TorBirdyOldPrefs.length; i++) {
+for (let i = 0; i < TorBirdyOldPrefs.length; i++) {
   var oldPref = TorBirdyOldPrefs[i];
   var setValue = kRestoreBranch + oldPref;
   var type = this.prefs.getPrefType(setValue);
@@ -502,14 +502,14 @@ TorBirdy.prototype = {
 }
 
 // Enable plugins.
-for(var i = 0; i < this.plugins.length; i++) {
+for(let i = 0; i < this.plugins.length; i++) {
 this.plugins[i].disabled = false;
 }
 
 // Now clear all TorBirdy preferences.
 var clearPrefs = Cc["@mozilla.org/preferences-service;1"]
  
.getService(Ci.nsIPrefService).getBranch(kTorBirdyBranch).getChildList("", {});
-for (var i = 0; i < clearPrefs.length; i++) {
+for (let i = 0; i < clearPrefs.length; i++) {
 this.prefs.clearUserPref(kTorBirdyBranch + clearPrefs[i]);
 }
   },
@@ -517,7 +517,7 @@ TorBirdy.prototype = {
   setPrefs: function() {
 // If custom values are set for specific preferences, override the 
defaults with them.
 // For each preference, get the type and then set the property.
-for (var i = 0; i < this.customPrefs.length; i++) {
+for (let i = 0; i < this.customPrefs.length; i++) {
   var typePref = this.prefs.getPrefType(this.customPrefs[i]);
   // String.
   if (typePref === 32) {
@@ -534,7 +534,7 @@ TorBirdy.prototype = {
   TorBirdyPrefs[this.customPrefs[i]] = value;
 }
 
-for (var each in TorBirdyPrefs) {
+for (let each in TorBirdyPrefs) {
   if (typeof TorBirdyPrefs[each] === "boolean") {
 this.prefs.setBoolPref(each, TorBirdyPrefs[each]);
   }
@@ -554,7 +554,7 @@ TorBirdy.prototype = {
   // First copy TorBirdyPrefs to TorBirdyOldPrefs.
   TorBirdyOldPrefs.push.apply(TorBirdyOldPrefs, 
Object.keys(TorBirdyPrefs));
 
-  for (var i = 0; i < TorBirdyOldPrefs.length; i++) {
+  for (let i = 0; i < TorBirdyOldPrefs.length; i++) {
 var oldPref = TorBirdyOldPrefs[i];
 var type = 

[tor-commits] [tor/master] Let's not even talk about those errors, ok?

[nickm]

commit 3cdc8bfa2c995b3b381e462128c0408e0ae5b3af
Author: Nick Mathewson 
Date:   Mon May 30 17:14:46 2016 -0400

Let's not even talk about those errors, ok?
---
 src/test/test_dir_handle_get.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c
index e82eddf..7f92c8c 100644
--- a/src/test/test_dir_handle_get.c
+++ b/src/test/test_dir_handle_get.c
@@ -1013,7 +1013,7 @@ test_dir_handle_get_server_descriptors_d(void* data)
   conn = dir_connection_new(tor_addr_family(_TOR_ADDR));
 
   char req_header[155]; /* XXX Why 155? What kind of number is that?? */
-  tor_snprintf(req_header, sizeof(req_handler),
+  tor_snprintf(req_header, sizeof(req_header),
SERVER_DESC_GET("d/%s+" HEX1 "+" HEX2), hex_digest);
   tt_int_op(directory_handle_command_get(conn, req_header, NULL, 0), OP_EQ, 0);
 
@@ -1358,7 +1358,7 @@ test_dir_handle_get_server_keys_fp(void* data)
 
   conn = dir_connection_new(tor_addr_family(_TOR_ADDR));
   char req[71];
-  tor_snprintf(req, (sizeof(req),
+  tor_snprintf(req, sizeof(req),
  GET("/tor/keys/fp/%s"), TEST_CERT_IDENT_KEY);
   tt_int_op(directory_handle_command_get(conn, req, NULL, 0), OP_EQ, 0);
 

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [tor/master] Wait, we had sprintf() in our unit tests?? FOR SHAME!

[nickm]

commit 97f2c1c58e0ad0a60b3c0ce35921e444c93b6e4f
Author: Nick Mathewson 
Date:   Mon May 30 16:50:57 2016 -0400

Wait, we had sprintf() in our unit tests?? FOR SHAME!
---
 src/test/test_dir_handle_get.c | 34 --
 1 file changed, 20 insertions(+), 14 deletions(-)

diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c
index 3029e98..e82eddf 100644
--- a/src/test/test_dir_handle_get.c
+++ b/src/test/test_dir_handle_get.c
@@ -393,7 +393,7 @@ 
test_dir_handle_get_rendezvous2_on_encrypted_conn_success(void *data)
   TO_CONN(conn)->linked = 1;
   tt_assert(connection_dir_is_encrypted(conn));
 
-  sprintf(req, RENDEZVOUS2_GET("%s"), desc_id_base32);
+  tor_snprintf(req, sizeof(req), RENDEZVOUS2_GET("%s"), desc_id_base32);
 
   tt_int_op(directory_handle_command_get(conn, req, NULL, 0), OP_EQ, 0);
 
@@ -408,7 +408,7 @@ 
test_dir_handle_get_rendezvous2_on_encrypted_conn_success(void *data)
   tt_assert(strstr(header, "Content-Type: text/plain\r\n"));
   tt_assert(strstr(header, "Content-Encoding: identity\r\n"));
   tt_assert(strstr(header, "Pragma: no-cache\r\n"));
-  sprintf(buff, "Content-Length: %ld\r\n", (long) body_len);
+  tor_snprintf(buff, sizeof(buff), "Content-Length: %ld\r\n", (long) body_len);
   tt_assert(strstr(header, buff));
 
   tt_int_op(body_used, OP_EQ, strlen(body));
@@ -520,7 +520,7 @@ test_dir_handle_get_micro_d(void *data)
   /* Make the request */
   conn = dir_connection_new(tor_addr_family(_TOR_ADDR));
 
-  sprintf(path, MICRODESC_GET("%s"), digest_base64);
+  tor_snprintf(path, sizeof(path), MICRODESC_GET("%s"), digest_base64);
   tt_int_op(directory_handle_command_get(conn, path, NULL, 0), OP_EQ, 0);
 
   fetch_from_buf_http(TO_CONN(conn)->outbuf, , MAX_HEADERS_SIZE,
@@ -590,7 +590,7 @@ test_dir_handle_get_micro_d_server_busy(void *data)
   /* Make the request */
   conn = dir_connection_new(tor_addr_family(_TOR_ADDR));
 
-  sprintf(path, MICRODESC_GET("%s"), digest_base64);
+  tor_snprintf(path, sizeof(path), MICRODESC_GET("%s"), digest_base64);
   tt_int_op(directory_handle_command_get(conn, path, NULL, 0), OP_EQ, 0);
 
   fetch_from_buf_http(TO_CONN(conn)->outbuf, , MAX_HEADERS_SIZE,
@@ -952,7 +952,8 @@ test_dir_handle_get_server_descriptors_fp(void* data)
DIGEST_LEN);
 
   char req[155];
-  sprintf(req, SERVER_DESC_GET("fp/%s+" HEX1 "+" HEX2), hex_digest);
+  tor_snprintf(req, sizeof(req), SERVER_DESC_GET("fp/%s+" HEX1 "+" HEX2),
+   hex_digest);
   tt_int_op(directory_handle_command_get(conn, req, NULL, 0), OP_EQ, 0);
 
   //TODO: Is this a BUG?
@@ -1011,8 +1012,9 @@ test_dir_handle_get_server_descriptors_d(void* data)
 
   conn = dir_connection_new(tor_addr_family(_TOR_ADDR));
 
-  char req_header[155];
-  sprintf(req_header, SERVER_DESC_GET("d/%s+" HEX1 "+" HEX2), hex_digest);
+  char req_header[155]; /* XXX Why 155? What kind of number is that?? */
+  tor_snprintf(req_header, sizeof(req_handler),
+   SERVER_DESC_GET("d/%s+" HEX1 "+" HEX2), hex_digest);
   tt_int_op(directory_handle_command_get(conn, req_header, NULL, 0), OP_EQ, 0);
 
   //TODO: Is this a BUG?
@@ -1080,8 +1082,9 @@ test_dir_handle_get_server_descriptors_busy(void* data)
 
   #define HEX1 "Fe0daff89127389bc67558691231234551193EEE"
   #define HEX2 "Deadbeef999119f00ba4"
-  char req_header[155];
-  sprintf(req_header, SERVER_DESC_GET("d/%s+" HEX1 "+" HEX2), hex_digest);
+  char req_header[155]; /* XXX 155? Why 155? */
+  tor_snprintf(req_header, sizeof(req_header),
+   SERVER_DESC_GET("d/%s+" HEX1 "+" HEX2), hex_digest);
   tt_int_op(directory_handle_command_get(conn, req_header, NULL, 0), OP_EQ, 0);
 
   fetch_from_buf_http(TO_CONN(conn)->outbuf, , MAX_HEADERS_SIZE,
@@ -1355,7 +1358,8 @@ test_dir_handle_get_server_keys_fp(void* data)
 
   conn = dir_connection_new(tor_addr_family(_TOR_ADDR));
   char req[71];
-  sprintf(req, GET("/tor/keys/fp/%s"), TEST_CERT_IDENT_KEY);
+  tor_snprintf(req, (sizeof(req),
+ GET("/tor/keys/fp/%s"), TEST_CERT_IDENT_KEY);
   tt_int_op(directory_handle_command_get(conn, req, NULL, 0), OP_EQ, 0);
 
   fetch_from_buf_http(TO_CONN(conn)->outbuf, , MAX_HEADERS_SIZE,
@@ -1427,7 +1431,8 @@ test_dir_handle_get_server_keys_sk(void* data)
 
   conn = dir_connection_new(tor_addr_family(_TOR_ADDR));
   char req[71];
-  sprintf(req, GET("/tor/keys/sk/%s"), TEST_SIGNING_KEY);
+  tor_snprintf(req, sizeof(req),
+   GET("/tor/keys/sk/%s"), TEST_SIGNING_KEY);
   tt_int_op(directory_handle_command_get(conn, req, NULL, 0), OP_EQ, 0);
 
   fetch_from_buf_http(TO_CONN(conn)->outbuf, , MAX_HEADERS_SIZE,
@@ -1510,8 +1515,9 @@ test_dir_handle_get_server_keys_fpsk(void* data)
   conn = dir_connection_new(tor_addr_family(_TOR_ADDR));
 
   char req[115];
-  sprintf(req, GET("/tor/keys/fp-sk/%s-%s"),
-  TEST_CERT_IDENT_KEY, TEST_SIGNING_KEY);
+  tor_snprintf(req, sizeof(req),
+   

[tor-commits] [tor/master] Replace nearly all XXX0vv comments with smarter ones

[nickm]

commit 4f1a04ff9c74456c03ac67adb7ae780e4e77c71e
Author: Nick Mathewson 
Date:   Mon May 30 16:18:16 2016 -0400

Replace nearly all XXX0vv comments with smarter ones

So, back long ago, XXX012 meant, "before Tor 0.1.2 is released, we
had better revisit this comment and fix it!"

But we have a huge pile of such comments accumulated for a large
number of released versions!  Not cool.

So, here's what I tried to do:

  * 0.2.9 and 0.2.8 are retained, since those are not yet released.

  * XXX+ or XXX++ or XXX or whatever means, "This one looks
quite important!"

  * The others, after one-by-one examination, are downgraded to
plain old XXX.  Which doesn't mean they aren't a problem -- just
that they cannot possibly be a release-blocking problem.
---
 src/common/address.c |  2 +-
 src/common/compat.c  |  3 ++-
 src/common/procmon.c |  9 +
 src/ext/eventdns.c   |  3 +--
 src/or/buffers.c |  6 +++---
 src/or/circpathbias.c|  5 ++---
 src/or/circuitbuild.c|  9 -
 src/or/circuituse.c  | 12 ++--
 src/or/config.c  | 12 +++-
 src/or/config.h  |  2 +-
 src/or/connection.c  | 10 +-
 src/or/connection_edge.c | 10 +-
 src/or/connection_or.c   |  2 +-
 src/or/control.c |  4 ++--
 src/or/directory.c   |  9 +
 src/or/dirserv.c |  6 +++---
 src/or/dirvote.c |  2 +-
 src/or/entrynodes.c  |  7 ---
 src/or/main.c|  2 +-
 src/or/microdesc.c   |  2 +-
 src/or/networkstatus.c   |  2 +-
 src/or/onion.c   |  2 +-
 src/or/or.h  | 10 +-
 src/or/rendclient.c  |  6 +++---
 src/or/rephist.c |  2 +-
 src/or/router.c  |  2 +-
 src/or/routerlist.c  |  6 +++---
 src/or/routerparse.c |  4 ++--
 src/or/transports.c  |  2 +-
 src/test/test_dir.c  |  2 +-
 30 files changed, 75 insertions(+), 80 deletions(-)

diff --git a/src/common/address.c b/src/common/address.c
index a6e0f3f..01607a4 100644
--- a/src/common/address.c
+++ b/src/common/address.c
@@ -1809,7 +1809,7 @@ MOCK_IMPL(smartlist_t *,get_interface_address6_list,(int 
severity,
 
 /* ==
  * IPv4 helpers
- * 024 IPv6 deprecate some of these.
+ *  IPv6 deprecate some of these.
  */
 
 /** Given an address of the form "ip:port", try to divide it into its
diff --git a/src/common/compat.c b/src/common/compat.c
index 23eaa13..9d49b8f 100644
--- a/src/common/compat.c
+++ b/src/common/compat.c
@@ -15,7 +15,8 @@
 /* This is required on rh7 to make strptime not complain.
  * We also need it to make memmem get defined (where available)
  */
-/* 024 We should just  use AC_USE_SYSTEM_EXTENSIONS in our autoconf,
+
+/*  We should just  use AC_USE_SYSTEM_EXTENSIONS in our autoconf,
  * and get this (and other important stuff!) automatically. Once we do that,
  * make sure to also change the extern char **environ detection in
  * configure.ac, because whether that is declared or not depends on whether
diff --git a/src/common/procmon.c b/src/common/procmon.c
index 12d53fc..4ecee26 100644
--- a/src/common/procmon.c
+++ b/src/common/procmon.c
@@ -116,11 +116,11 @@ struct tor_process_monitor_t {
* periodically check whether the process we have a handle to has
* ended. */
   HANDLE hproc;
-  /* XXX023 We can and should have Libevent watch hproc for us,
-   * if/when some version of Libevent 2.x can be told to do so. */
+  /*  We should have Libevent watch hproc for us,
+   * if/when some version of Libevent can be told to do so. */
 #endif
 
-  /* XXX023 On Linux, we can and should receive the 22nd
+  /*  On Linux, we can and should receive the 22nd
* (space-delimited) field (‘starttime’) of /proc/$PID/stat from the
* owning controller and store it, and poll once in a while to see
* whether it has changed -- if so, the kernel has *definitely*
@@ -130,7 +130,8 @@ struct tor_process_monitor_t {
* systems whose admins have mounted procfs, or the start-time field
* of the process-information structure returned by kvmgetprocs() on
* any system.  The latter is ickier. */
-  /* XXX023 On FreeBSD (and possibly other kqueue systems), we can and
+
+  /*  On FreeBSD (and possibly other kqueue systems), we can and
* should arrange to receive EVFILT_PROC NOTE_EXIT notifications for
* pid, so we don't have to do such a heavyweight poll operation in
* order to avoid the PID-reassignment race condition.  (We would
diff --git a/src/ext/eventdns.c b/src/ext/eventdns.c
index fc5657c..7046f32 100644
--- a/src/ext/eventdns.c
+++ b/src/ext/eventdns.c
@@ -2004,8 +2004,7 @@ evdns_request_timeout_callback(int fd, short events, void 
*arg) {
} else {
/* retransmit it */
/* Stop waiting for the timeout.  No need to do this in
-* request_finished; that one already deletes 

[tor-commits] [tor/master] We no longer generate v0 directories. Remove the code to do so

[nickm]

commit ce31db4326599c3f86307fca5286349150cb871e
Author: Nick Mathewson 
Date:   Mon May 30 16:05:37 2016 -0400

We no longer generate v0 directories. Remove the code to do so
---
 src/or/dirserv.c | 88 
 src/or/dirserv.h |  2 --
 2 files changed, 90 deletions(-)

diff --git a/src/or/dirserv.c b/src/or/dirserv.c
index d1ea5df..e61f56e 100644
--- a/src/or/dirserv.c
+++ b/src/or/dirserv.c
@@ -985,94 +985,6 @@ router_is_active(const routerinfo_t *ri, const node_t 
*node, time_t now)
   return 1;
 }
 
-/** Generate a new v1 directory and write it into a newly allocated string.
- * Point *dir_out to the allocated string.  Sign the
- * directory with private_key.  Return 0 on success, -1 on
- * failure. If complete is set, give us all the descriptors;
- * otherwise leave out non-running and non-valid ones.
- */
-int
-dirserv_dump_directory_to_string(char **dir_out,
- crypto_pk_t *private_key)
-{
-  /*  024 Get rid of this function if we can confirm that nobody's
-   * fetching these any longer */
-  char *cp;
-  char *identity_pkey; /* Identity key, DER64-encoded. */
-  char *recommended_versions;
-  char digest[DIGEST_LEN];
-  char published[ISO_TIME_LEN+1];
-  char *buf = NULL;
-  size_t buf_len;
-  size_t identity_pkey_len;
-  time_t now = time(NULL);
-
-  tor_assert(dir_out);
-  *dir_out = NULL;
-
-  if (crypto_pk_write_public_key_to_string(private_key,_pkey,
-   _pkey_len)<0) {
-log_warn(LD_BUG,"write identity_pkey to string failed!");
-return -1;
-  }
-
-  recommended_versions =
-format_versions_list(get_options()->RecommendedVersions);
-
-  format_iso_time(published, now);
-
-  buf_len = 2048+strlen(recommended_versions);
-
-  buf = tor_malloc(buf_len);
-  /* We'll be comparing against buf_len throughout the rest of the
- function, though strictly speaking we shouldn't be able to exceed
- it.  This is C, after all, so we may as well check for buffer
- overruns.*/
-
-  tor_snprintf(buf, buf_len,
-   "signed-directory\n"
-   "published %s\n"
-   "recommended-software %s\n"
-   "router-status %s\n"
-   "dir-signing-key\n%s\n",
-   published, recommended_versions, "",
-   identity_pkey);
-
-  tor_free(recommended_versions);
-  tor_free(identity_pkey);
-
-  cp = buf + strlen(buf);
-  *cp = '\0';
-
-  /* These multiple strlcat calls are inefficient, but dwarfed by the RSA
- signature. */
-  if (strlcat(buf, "directory-signature ", buf_len) >= buf_len)
-goto truncated;
-  if (strlcat(buf, get_options()->Nickname, buf_len) >= buf_len)
-goto truncated;
-  if (strlcat(buf, "\n", buf_len) >= buf_len)
-goto truncated;
-
-  if (router_get_dir_hash(buf,digest)) {
-log_warn(LD_BUG,"couldn't compute digest");
-tor_free(buf);
-return -1;
-  }
-  note_crypto_pk_op(SIGN_DIR);
-  if (router_append_dirobj_signature(buf,buf_len,digest,DIGEST_LEN,
- private_key)<0) {
-tor_free(buf);
-return -1;
-  }
-
-  *dir_out = buf;
-  return 0;
- truncated:
-  log_warn(LD_BUG,"tried to exceed string length.");
-  tor_free(buf);
-  return -1;
-}
-
 //
 
 /* A set of functions to answer questions about how we'd like to behave
diff --git a/src/or/dirserv.h b/src/or/dirserv.h
index 9a9725a..3c914e9 100644
--- a/src/or/dirserv.h
+++ b/src/or/dirserv.h
@@ -47,8 +47,6 @@ enum was_router_added_t dirserv_add_descriptor(routerinfo_t 
*ri,
 void dirserv_set_router_is_running(routerinfo_t *router, time_t now);
 int list_server_status_v1(smartlist_t *routers, char **router_status_out,
   int for_controller);
-int dirserv_dump_directory_to_string(char **dir_out,
- crypto_pk_t *private_key);
 char *dirserv_get_flag_thresholds_line(void);
 void dirserv_compute_bridge_flag_thresholds(void);
 

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [translation/tor-messenger-fingerdtd] Update translations for tor-messenger-fingerdtd

[translation]

commit 88390a939a5b56de115f88daa66ee6f690722d87
Author: Translation commit bot 
Date:   Mon May 30 19:46:22 2016 +

Update translations for tor-messenger-fingerdtd
---
 pl/finger.dtd | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pl/finger.dtd b/pl/finger.dtd
index bd371ca..5943e49 100644
--- a/pl/finger.dtd
+++ b/pl/finger.dtd
@@ -12,4 +12,4 @@
 
 
 
-
\ No newline at end of file
+
\ No newline at end of file

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [translation/tor-messenger-fingerdtd_completed] Update translations for tor-messenger-fingerdtd_completed

[translation]

commit 53a12b4dccc1d3445c09978bfc493162e80311ec
Author: Translation commit bot 
Date:   Mon May 30 19:46:25 2016 +

Update translations for tor-messenger-fingerdtd_completed
---
 pl/finger.dtd | 7 ++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/pl/finger.dtd b/pl/finger.dtd
index fa58403..5943e49 100644
--- a/pl/finger.dtd
+++ b/pl/finger.dtd
@@ -7,4 +7,9 @@
 
 
 
-
\ No newline at end of file
+
+
+
+
+
+
\ No newline at end of file

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [tor/master] remove now-irrelevant XXX020 comments in configure.ac

[nickm]

commit 57bf8bb26357444219c07d57caa9260a75ff7681
Author: Nick Mathewson 
Date:   Mon May 30 15:31:19 2016 -0400

remove now-irrelevant XXX020 comments in configure.ac

They apply to ancient GCC versions and to an unknown set of
configuration options. Notabug.
---
 configure.ac | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/configure.ac b/configure.ac
index 26f85b5..7259bf7 100644
--- a/configure.ac
+++ b/configure.ac
@@ -30,7 +30,6 @@ fi
 # the += operator on it in src/or/Makefile.am
 CPPFLAGS="$CPPFLAGS -I\${top_srcdir}/src/common"
 
-#020 We should make these enabled or not, before 0.2.0.x-final
 AC_ARG_ENABLE(openbsd-malloc,
AS_HELP_STRING(--enable-openbsd-malloc, [use malloc code from OpenBSD.  
Linux only]))
 AC_ARG_ENABLE(static-openssl,
@@ -1710,7 +1709,6 @@ if test "x$enable_gcc_warnings_advisory" != "xno"; then
 
   if test "x$have_gcc42" = "xyes"; then
 # These warnings break gcc 4.0.2 and work on gcc 4.2
-# 020 See if any of these work with earlier versions.
 CFLAGS="$CFLAGS -Waddress -Wmissing-noreturn -Wstrict-overflow=1"
 
 # We used to use -Wstrict-overflow=5, but that breaks us heavily under 4.3.
@@ -1723,7 +1721,6 @@ if test "x$enable_gcc_warnings_advisory" != "xno"; then
 
   if test "x$have_gcc43" = "xyes"; then
 # These warnings break gcc 4.2 and work on gcc 4.3
-# 020 See if any of these work with earlier versions.
 CFLAGS="$CFLAGS -Wextra -Warray-bounds"
   fi
 

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [webwml/master] Add new Tor Browser version: 6.0

[boklm]

commit 9f11ba3c13dd68089b942cb2dbd3b2804e8a3e3c
Author: Nicolas Vigier 
Date:   Mon May 30 20:56:17 2016 +0200

Add new Tor Browser version: 6.0
---
 include/versions.wmi   | 22 +++---
 projects/torbrowser/RecommendedTBBVersions |  4 
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/include/versions.wmi b/include/versions.wmi
index 750b575..e7c4b2c 100644
--- a/include/versions.wmi
+++ b/include/versions.wmi
@@ -4,27 +4,27 @@
 0.2.7.6
 
 maint-5.5
-5.5.5
+6.0
 6.0a5
 
-5.5.5
-2016-04-26
+6.0
+2016-05-30
 6.0a5
 2016-04-28
 
-5.5.5
-2016-04-26
-5.5.5
-2016-04-26
+6.0
+2016-05-30
+6.0
+2016-05-30
 6.0a5
 2016-04-28
 6.0a5
 2016-04-28
 
-5.5.5
-2016-04-26
-5.5.5
-2016-04-26
+6.0
+2016-05-30
+6.0
+2016-05-30
 6.0a5
 2016-04-28
 
diff --git a/projects/torbrowser/RecommendedTBBVersions 
b/projects/torbrowser/RecommendedTBBVersions
index 61ac009..7b9dc5e 100644
--- a/projects/torbrowser/RecommendedTBBVersions
+++ b/projects/torbrowser/RecommendedTBBVersions
@@ -3,6 +3,10 @@
 "5.5.5-Linux",
 "5.5.5-MacOS",
 "5.5.5-Windows",
+"6.0",
+"6.0-Linux",
+"6.0-MacOS",
+"6.0-Windows",
 "6.0a5",
 "6.0a5-Linux",
 "6.0a5-MacOS",

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Address feedback by @bassosimone

[art]

commit 2d5b7988c23697cd6af390c6ed649d69b7380dff
Author: Arturo Filastò 
Date:   Tue May 24 11:34:58 2016 +0200

Address feedback by @bassosimone

* Distinguish between http-failure and http-diff

* Update description of web_connectivity test

* Various consistency and stylistic improvements
---
 ooni/nettests/blocking/web_connectivity.py | 23 ---
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/ooni/nettests/blocking/web_connectivity.py 
b/ooni/nettests/blocking/web_connectivity.py
index 4f5f4e4..3560cb0 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -68,8 +68,10 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 Web connectivity
 """
 name = "Web connectivity"
-description = ("Performs a HTTP GET request over Tor and one over the "
-  "local network and compares the two results.")
+description = ("Identifies the reason for blocking of a given URL by "
+   "performing DNS resolution of the hostname, doing a TCP "
+   "connect to the resolved IPs and then fetching the page "
+   "and comparing all these results with those of a control.")
 author = "Arturo Filastò"
 version = "0.1.0"
 
@@ -126,12 +128,12 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 if not l:
 continue
 # Skip comment lines
-elif l.startswith('#'):
+if l.startswith('#'):
 continue
 yield l
 
+fh = open(filename)
 try:
-fh = open(filename)
 line = fh.readline()
 # Detect the line of the citizenlab input file
 if line.startswith("url,"):
@@ -140,7 +142,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 fh.seek(0)
 generator = simple_file_generator(fh)
 for i in generator:
-if not i.startswith("http"):
+if (not i.startswith("http://;) and
+not i.startswith("https://;)):
 i = "http://{}/".format(i)
 yield i
 finally:
@@ -200,7 +203,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 log.msg("* doing DNS query for {}".format(self.hostname))
 return self.performALookup(self.hostname)
 
-def tcp_connect(self, socket):
+def experiment_tcp_connect(self, socket):
 log.msg("* connecting to {}".format(socket))
 ip_address, port = socket.split(":")
 port = int(port)
@@ -365,12 +368,10 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 experiment_http_failure is not None):
 blocking = 'tcp_ip'
 
-# XXX we may want to have different codes for these two types of
-# blocking
 elif (dns_consistent == True and
   tcp_connect == True and
   got_expected_web_page == False):
-blocking = 'http'
+blocking = 'http-diff'
 
 elif (dns_consistent == True and
   tcp_connect == True and
@@ -379,7 +380,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 if experiment_http_failure == 'dns_lookup_error':
 blocking = 'dns'
 else:
-blocking = 'http'
+blocking = 'http-failure'
 
 elif (dns_consistent == False and
   (got_expected_web_page == False or
@@ -425,7 +426,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 # STEALTH in here we should make changes to make the test more stealth
 dl = []
 for socket in sockets:
-dl.append(self.tcp_connect(socket))
+dl.append(self.experiment_tcp_connect(socket))
 results = yield defer.DeferredList(dl)
 
 experiment_http = self.experiment_http_get_request()



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Various improvement related to backend_client

[art]

commit 24ea012b508526165b1a7d1eef1d76475bd313e6
Author: Arturo Filastò 
Date:   Tue May 17 20:02:09 2016 +0200

Various improvement related to backend_client

* Include web_connectivity in default generated deck

* Fix serialisation of HTTP Headers

* Support setting collector and bouncer in oonireport upload
---
 ooni/deck.py|  4 +++-
 ooni/deckgen/cli.py | 22 +++---
 ooni/report/cli.py  |  2 +-
 ooni/report/tool.py | 30 +-
 ooni/reporter.py|  3 ++-
 ooni/templates/httpt.py |  2 +-
 6 files changed, 35 insertions(+), 28 deletions(-)

diff --git a/ooni/deck.py b/ooni/deck.py
index 007097e..404a4d0 100644
--- a/ooni/deck.py
+++ b/ooni/deck.py
@@ -136,10 +136,12 @@ class Deck(InputFile):
 net_test_loader = NetTestLoader(test['options']['subargs'],
 
annotations=test['options'].get('annotations', {}),
 test_file=nettest_path)
-if test['options']['collector']:
+if test['options']['collector'] is not None:
 net_test_loader.collector = CollectorClient(
 test['options']['collector']
 )
+if test['options']['bouncer'] is not None:
+self.bouncer = test['options']['bouncer']
 self.insert(net_test_loader)
 
 def insert(self, net_test_loader):
diff --git a/ooni/deckgen/cli.py b/ooni/deckgen/cli.py
index 2e0e976..a6046f3 100644
--- a/ooni/deckgen/cli.py
+++ b/ooni/deckgen/cli.py
@@ -27,6 +27,7 @@ class Options(usage.Options):
  "generate the deck."],
 ["collector", None, None, "Specify a custom collector to use when "
   "submitting reports"],
+["bouncer", None, None, "Specify a custom bouncer to use"],
 ["output", "o", None,
  "Specify the directory where to write output."]
 ]
@@ -44,7 +45,6 @@ class Deck(object):
 "annotations": None,
 
 "collector": None,
-# XXX setting this is currently not supported
 "bouncer": None,
 
 "reportfile": None,
@@ -56,13 +56,15 @@ class Deck(object):
 }
 }
 
-def __init__(self, collector=None):
+def __init__(self, collector=None, bouncer=None):
 self.deck_entries = []
 self.collector = collector
+self.bouncer = bouncer
 
 def add_test(self, test_file, subargs=[]):
 deck_entry = copy.deepcopy(self._base_entry)
 deck_entry['options']['collector'] = self.collector
+deck_entry['options']['bouncer'] = self.bouncer
 deck_entry['options']['test_file'] = test_file
 deck_entry['options']['subargs'] = subargs
 self.deck_entries.append(deck_entry)
@@ -89,24 +91,14 @@ def generate_deck(options):
 url_list_global = citizenlab_test_lists.generate_global_input(
 options['output']
 )
-dns_servers = namebench_dns_servers.generate_country_input(
-options['country-code'],
-options['output']
-)
 
-deck = Deck(collector=options['collector'])
+deck = Deck(collector=options['collector'], bouncer=options['bouncer'])
 deck.add_test('manipulation/http_invalid_request_line')
 deck.add_test('manipulation/http_header_field_manipulation')
 
 if url_list_country is not None:
-deck.add_test('blocking/http_requests', ['-f', url_list_country])
-deck.add_test('blocking/http_requests', ['-f', url_list_global])
-
-if url_list_country is not None:
-deck.add_test('blocking/dns_consistency',
-  ['-f', url_list_country, '-T', dns_servers])
-deck.add_test('blocking/dns_consistency',
-  ['-f', url_list_global, '-T', dns_servers])
+deck.add_test('blocking/web_connectivity', ['-f', url_list_country])
+deck.add_test('blocking/web_connectivity', ['-f', url_list_global])
 
 if config.advanced.debug:
 deck.pprint()
diff --git a/ooni/report/cli.py b/ooni/report/cli.py
index c1b4a22..e52b022 100644
--- a/ooni/report/cli.py
+++ b/ooni/report/cli.py
@@ -20,7 +20,7 @@ class Options(usage.Options):
  "Specify the configuration file to use."],
 ["collector", "c", None,
  "Specify the collector to upload the result to."],
-["bouncer", "b", "httpo://nkvphnp3p6agi5qq.onion",
+["bouncer", "b", None,
  "Specify the bouncer to query for a collector."]
 ]
 
diff --git a/ooni/report/tool.py b/ooni/report/tool.py
index fd504a6..d80a8e0 100644
--- a/ooni/report/tool.py
+++ b/ooni/report/tool.py
@@ -9,12 +9,15 @@ from ooni.reporter import OONIBReporter, OONIBReportLog
 from ooni.utils import log
 from ooni.report import parser
 from ooni.settings import config
-from ooni.backend_client import BouncerClient
+from ooni.backend_client import BouncerClient, CollectorClient
 
 
 

[tor-commits] [ooni-probe/master] Expand the heuristics for detecting blockpages in HTTP response

[art]

commit 7f8021efd69d28beb034a7cd25c60ea2c0016bf6
Author: Arturo Filastò 
Date:   Tue May 24 18:06:00 2016 +0200

Expand the heuristics for detecting blockpages in HTTP response

* Extract the title of the response

* Consider only common HTTP headers
---
 ooni/nettests/blocking/web_connectivity.py | 76 ++
 ooni/utils/net.py  | 39 +++
 2 files changed, 95 insertions(+), 20 deletions(-)

diff --git a/ooni/nettests/blocking/web_connectivity.py 
b/ooni/nettests/blocking/web_connectivity.py
index e640a5f..8f048e1 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -1,7 +1,6 @@
 # -*- encoding: utf-8 -*-
 
 import csv
-import json
 from urlparse import urlparse
 
 from ipaddr import IPv4Address, AddressValueError
@@ -20,7 +19,7 @@ from ooni.utils import log
 
 from ooni.backend_client import WebConnectivityClient
 
-from ooni.utils.net import StringProducer, BodyReceiver
+from ooni.utils.net import COMMON_SERVER_HEADERS, extract_title
 from ooni.templates import httpt, dnst
 from ooni.errors import failureToString
 
@@ -50,6 +49,7 @@ class UsageOptions(usage.Options):
 ['url', 'u', None, 'Specify a single URL to test'],
 ['dns-discovery', 'd', 'whoami.akamai.net', 'Specify the dns discovery 
test helper'],
 ['backend', 'b', None, 'The web_consistency backend test helper'],
+['retries', 'r', 1, 'Number of retries for the HTTP request'],
 ]
 
 
@@ -158,6 +158,12 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 if not self.input:
 raise Exception("No input specified")
 
+try:
+self.localOptions['retries'] = int(self.localOptions['retries'])
+except ValueError:
+self.localOptions['retries'] = 2
+
+self.report['retries'] = self.localOptions['retries']
 self.report['client_resolver'] = self.resolverIp
 self.report['dns_consistency'] = None
 self.report['body_length_match'] = None
@@ -188,7 +194,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 'body_length': -1,
 'failure': None,
 'status_code': -1,
-'headers': {}
+'headers': {},
+'title': ''
 }
 }
 if isinstance(self.localOptions['backend'], dict):
@@ -240,24 +247,36 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 )
 self.report['control'] = self.control
 
+@defer.inlineCallbacks
 def experiment_http_get_request(self):
-return self.doRequest(self.input, headers=REQUEST_HEADERS)
+retries = 0
+while True:
+try:
+result = yield self.doRequest(self.input,
+  headers=REQUEST_HEADERS)
+break
+except:
+if self.localOptions['retries'] > retries:
+raise
+retries += 1
+
+defer.returnValue(result)
 
 def compare_headers(self, experiment_http_response):
-count = 0
 control_headers_lower = {k.lower(): v for k, v in
-self.report['control']['http_request']['headers'].items()}
+self.report['control']['http_request']['headers'].items()
+}
+experiment_headers_lower = {k.lower(): v for k, v in
+experiment_http_response.headers.getAllRawHeaders()
+}
 
-for header_name, header_value in \
-experiment_http_response.headers.getAllRawHeaders():
-try:
-control_headers_lower[header_name.lower()]
-except KeyError:
-log.debug("Did not find the key {}".format(header_name))
-return False
-count += 1
+uncommon_ctrl_headers = (set(control_headers_lower.keys()) -
+ set(COMMON_SERVER_HEADERS))
+uncommon_exp_headers = (set(experiment_headers_lower.keys()) -
+set(COMMON_SERVER_HEADERS))
 
-return count == len(self.report['control']['http_request']['headers'])
+return len(uncommon_ctrl_headers.intersection(
+uncommon_exp_headers)) > 0
 
 def compare_body_lengths(self, experiment_http_response):
 control_body_length = self.control['http_request']['body_length']
@@ -279,6 +298,17 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 else:
 return False
 
+def compare_titles(self, experiment_http_response):
+experiment_title = extract_title(experiment_http_response.body).strip()
+control_title = self.control['http_request']['title'].strip()
+first_exp_word = experiment_title.split(' ')[0]
+first_ctrl_word = control_title.split(' ')[0]
+if len(first_exp_word) < 5:
+  

[tor-commits] [ooni-probe/master] Don't require the bouncer key to be present

[art]

commit 156ce3f630eb58a061841dfa6d6f6f6cf6755d83
Author: Arturo Filastò 
Date:   Tue May 17 20:24:15 2016 +0200

Don't require the bouncer key to be present
---
 ooni/deck.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ooni/deck.py b/ooni/deck.py
index 404a4d0..7a3b697 100644
--- a/ooni/deck.py
+++ b/ooni/deck.py
@@ -136,11 +136,11 @@ class Deck(InputFile):
 net_test_loader = NetTestLoader(test['options']['subargs'],
 
annotations=test['options'].get('annotations', {}),
 test_file=nettest_path)
-if test['options']['collector'] is not None:
+if test['options'].get('collector', None) is not None:
 net_test_loader.collector = CollectorClient(
 test['options']['collector']
 )
-if test['options']['bouncer'] is not None:
+if test['options'].get('bouncer', None) is not None:
 self.bouncer = test['options']['bouncer']
 self.insert(net_test_loader)
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Update to version 1.5.0

[art]

commit 5fdaa3ebe0928bbbc8cd8088d160cf1a40dd9730
Author: Arturo Filastò 
Date:   Mon May 30 17:53:28 2016 +0200

Update to version 1.5.0
---
 ChangeLog.rst| 10 ++
 ooni/__init__.py |  2 +-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 56dd061..5470570 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,6 +1,16 @@
 Changelog
 =
 
+v1.5.0 (Mon, 30 May 2016)
+-
+codename: The Big Wave
+
+* Implement web_connectivity test that measures for both DNS and HTTP
+  censorship.
+
+* Fix a regression bug that lead to Tor exit ip address not being included in
+  reports.
+
 v1.4.2 (Fri, 29 Apr 2016)
 -
 
diff --git a/ooni/__init__.py b/ooni/__init__.py
index e02b430..59c0143 100644
--- a/ooni/__init__.py
+++ b/ooni/__init__.py
@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 
 __author__ = "Open Observatory of Network Interference"
-__version__ = "1.5.0.dev0"
+__version__ = "1.5.0"
 # This is the version number of resources to be downloaded
 # when a release is made it should be aligned to __version__
 __resources_version__ = "1.4.2"

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Match status codes only when the control status code is not an error

[art]

commit cd36a0d059b88810695a1b952ab54e853253f17e
Author: Arturo Filastò 
Date:   Wed May 25 12:35:22 2016 +0200

Match status codes only when the control status code is not an error
---
 ooni/nettests/blocking/web_connectivity.py | 13 +++--
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/ooni/nettests/blocking/web_connectivity.py 
b/ooni/nettests/blocking/web_connectivity.py
index 8f048e1..7692d70 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -317,10 +317,11 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 self.report['headers_match'] = \
 self.compare_headers(experiment_http_response)
 
-self.report['status_code_match'] = (
-experiment_http_response.code ==
-self.control['http_request']['status_code']
-)
+if str(self.control['http_request']['status_code'])[0] != '5':
+self.report['status_code_match'] =  (
+self.control['http_request']['status_code'] ==
+experiment_http_response.code
+)
 
 self.report['title_match'] = 
self.compare_titles(experiment_http_response)
 
@@ -397,8 +398,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 got_expected_web_page = (
 (self.report['body_length_match'] is True or
  self.report['headers_match'] is True or
- self.report['title_match'])
-and self.report['status_code_match'] is True
+ self.report['title_match'] is True)
+and self.report['status_code_match'] is not False
 )
 
 if (dns_consistent == True and tcp_connect == False and



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Add tor to the apt-get command.

[art]

commit 07eef0862220b7a1d7f651e2ce5511707cdc1d4a
Author: Arturo Filastò 
Date:   Sat May 21 14:07:49 2016 +0200

Add tor to the apt-get command.
---
 README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index 5227cc6..1c2ec82 100644
--- a/README.rst
+++ b/README.rst
@@ -109,7 +109,7 @@ Make sure you have installed the following dependencies:
 
 On debian based systems this can generally be done by running::
 
-sudo apt-get install -y build-essential libdumbnet-dev libpcap-dev 
libgeoip-dev libffi-dev python-dev python-pip
+sudo apt-get install -y build-essential libdumbnet-dev libpcap-dev 
libgeoip-dev libffi-dev python-dev python-pip tor
 
 Then you should be able to install ooniprobe by running::
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Address feedback by @bassosimone

[art]

commit 8e7199d85c877353b83b3da98936c602d4c4a5b4
Author: Arturo Filastò 
Date:   Mon May 30 10:42:29 2016 +0200

Address feedback by @bassosimone

* Test the next words if the word is less than 5 chars in web_connectivity
  test.

* Move the charset detection charset into the correct module and change the
  unittest accordingly.

* Include ooni.common in setup.py

* Various code style fixes
---
 ooni/backend_client.py |  6 +-
 ooni/common/http_utils.py  |  2 +-
 ooni/nettests/blocking/web_connectivity.py | 21 -
 ooni/templates/httpt.py|  4 
 ooni/tests/test_common.py  | 19 +++
 ooni/tests/test_templates.py   | 16 
 ooni/utils/net.py  |  1 -
 setup.py   |  1 +
 8 files changed, 34 insertions(+), 36 deletions(-)

diff --git a/ooni/backend_client.py b/ooni/backend_client.py
index d4c463e..0e85dd7 100644
--- a/ooni/backend_client.py
+++ b/ooni/backend_client.py
@@ -207,9 +207,7 @@ class CollectorClient(OONIBClient):
 @d.addErrback
 def err(failure):
 failure.trap(Error)
-if failure.value.status == '404':
-return True
-return False
+return failure.value.status == '404'
 
 return d
 
@@ -345,8 +343,6 @@ class CollectorClient(OONIBClient):
 
 class WebConnectivityClient(OONIBClient):
 def isReachable(self):
-# XXX maybe in the future we can have a dedicated API endpoint to
-# test the reachability of the collector.
 d = self.queryBackend('GET', '/status')
 
 @d.addCallback
diff --git a/ooni/common/http_utils.py b/ooni/common/http_utils.py
index 6d636d5..57c3a15 100644
--- a/ooni/common/http_utils.py
+++ b/ooni/common/http_utils.py
@@ -2,7 +2,7 @@ import re
 import codecs
 from base64 import b64encode
 
-META_CHARSET_REGEXP = 
re.compile(']*?charset\s*=[\s"\']*([^\s"\'/>]*)')
+META_CHARSET_REGEXP = 
re.compile(']*?charset\s*=[\s"\']*([^\s"\'/>!;]+)')
 
 def representBody(body):
 if not body:
diff --git a/ooni/nettests/blocking/web_connectivity.py 
b/ooni/nettests/blocking/web_connectivity.py
index 655bf76..b4ccff1 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -42,10 +42,9 @@ class UsageOptions(usage.Options):
 def is_public_ipv4_address(address):
 try:
 ip_address = IPv4Address(address)
-if not any([ip_address.is_private,
-ip_address.is_loopback]):
-return True
-return False
+return not any(
+[ip_address.is_private, ip_address.is_loopback]
+)
 except AddressValueError:
 return None
 
@@ -293,13 +292,17 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 def compare_titles(self, experiment_http_response):
 experiment_title = extractTitle(experiment_http_response.body).strip()
 control_title = self.control['http_request']['title'].strip()
-first_exp_word = experiment_title.split(' ')[0]
-first_ctrl_word = control_title.split(' ')[0]
-if len(first_exp_word) < 5:
+
+control_words = control_title.split(' ')
+for exp_word, idx in enumerate(experiment_title.split(' ')):
 # We don't consider to match words that are shorter than 5
 # characters (5 is the average word length for english)
-return False
-return (first_ctrl_word.lower() == first_exp_word.lower())
+if len(exp_word) < 5:
+continue
+try:
+return control_words[idx].lower() == exp_word.lower()
+except IndexError:
+return False
 
 def compare_http_experiments(self, experiment_http_response):
 
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index f8ea941..3f7e77d 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -1,5 +1,3 @@
-import re
-import codecs
 import random
 
 from txtorcon.interface import StreamListenerMixin
@@ -22,8 +20,6 @@ from ooni.common.txextra import FixedRedirectAgent, 
TrueHeadersAgent
 from ooni.common.http_utils import representBody
 from ooni.errors import handleAllFailures
 
-META_CHARSET_REGEXP = 
re.compile(']*?charset\s*=[\s"\']*([^\s"\'/>!;]+)')
-
 class InvalidSocksProxyOption(Exception):
 pass
 
diff --git a/ooni/tests/test_common.py b/ooni/tests/test_common.py
new file mode 100644
index 000..1cd77cf
--- /dev/null
+++ b/ooni/tests/test_common.py
@@ -0,0 +1,19 @@
+from twisted.trial import unittest
+from ooni.common.http_utils import META_CHARSET_REGEXP
+
+class TestHTTPUtils(unittest.TestCase):
+def test_charset_detection(self):
+no_charset_html 

[tor-commits] [ooni-probe/master] Merge pull request #514 from TheTorProject/feature/web_connectivity

[art]

commit 1c030e17e5390e3417d7582a5b3613c8696e53b1
Merge: a62a614 c046653
Author: Arturo Filastò 
Date:   Mon May 30 17:39:59 2016 +0200

Merge pull request #514 from TheTorProject/feature/web_connectivity

Feature/web connectivity

 bin/oonireport |   2 +-
 data/ooniprobe.conf.sample |   2 +-
 ooni/backend_client.py | 365 ++
 ooni/common/__init__.py|  10 +
 ooni/common/http_utils.py  |  55 ++
 ooni/common/tcp_utils.py   |  10 +
 ooni/common/txextra.py | 202 
 ooni/deck.py   | 234 ++---
 ooni/deckgen/cli.py|  22 +-
 ooni/director.py   |  51 +-
 ooni/errors.py |  42 +-
 ooni/geoip.py  |  25 +-
 ooni/nettest.py|  23 +-
 ooni/nettests/blocking/web_connectivity.py | 554 +
 .../manipulation/http_header_field_manipulation.py |   2 +-
 ooni/oonibclient.py| 221 
 ooni/oonicli.py|  43 +-
 ooni/report/cli.py |   2 +-
 ooni/report/tool.py|  32 +-
 ooni/reporter.py   | 270 +++---
 ooni/settings.py   |   2 +-
 ooni/tasks.py  |   5 +-
 ooni/templates/httpt.py|  95 ++--
 ooni/templates/scapyt.py   |   8 +-
 ooni/tests/mocks.py|  38 +-
 ooni/tests/test_common.py  |  19 +
 ooni/tests/test_deck.py|  25 +-
 ooni/tests/test_director.py|   2 +
 ooni/tests/test_nettest.py |  13 +-
 ooni/tests/test_onion.py   |  16 +
 ooni/tests/test_oonibclient.py |  47 +-
 ooni/tests/test_oonicli.py |  10 +-
 ooni/tests/test_reporter.py|  32 +-
 ooni/tests/test_templates.py   |  14 +-
 ooni/tests/test_trueheaders.py |   2 +-
 ooni/tests/test_utils.py   |  34 +-
 ooni/utils/__init__.py |  64 +--
 ooni/utils/log.py  |   2 +-
 ooni/utils/net.py  |  54 +-
 ooni/utils/onion.py|   8 +
 ooni/utils/socks.py|  25 +
 ooni/utils/trueheaders.py  | 181 ---
 setup.py   |   1 +
 43 files changed, 1858 insertions(+), 1006 deletions(-)




___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Inject the correct url when following redirects

[art]

commit 6380468033cd7cca542184ebe8d2b4148aaec2b7
Author: Arturo Filastò 
Date:   Tue May 24 18:25:40 2016 +0200

Inject the correct url when following redirects
---
 ooni/templates/httpt.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index ca0306b..6e0154c 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -196,6 +196,9 @@ class HTTPTest(NetTestCase):
 if (config.privacy.includeip is False and config.probe_ip.address 
is not None and
 (isinstance(response_body, str) or 
isinstance(response_body, unicode))):
 response_body = response_body.replace(config.probe_ip.address, 
"[REDACTED]")
+if (getattr(response, 'request', None) and
+getattr(response.request, 'absoluteURI', None)):
+session['request']['url'] = response.request.absoluteURI
 session['response'] = {
 'headers': _representHeaders(response.headers),
 'body': response_body,



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Write a msg when we perform the control lookup

[art]

commit c046653fbd63ff6beb35aa1d87d0bbe5e0014fef
Author: Arturo Filastò 
Date:   Mon May 30 15:18:31 2016 +0200

Write a msg when we perform the control lookup

Be more lax when detecting header key similarity
---
 ooni/nettests/blocking/web_connectivity.py | 5 +
 1 file changed, 5 insertions(+)

diff --git a/ooni/nettests/blocking/web_connectivity.py 
b/ooni/nettests/blocking/web_connectivity.py
index b4ccff1..0dba239 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -229,6 +229,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 
 @defer.inlineCallbacks
 def control_request(self, sockets):
+log.msg("* performing control request with backend")
 self.control = yield self.web_connectivity_client.control(
 http_request=self.input,
 tcp_connect=sockets
@@ -261,6 +262,10 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 experiment_http_response.headers.getAllRawHeaders()
 }
 
+if (set(control_headers_lower.keys()) ==
+set(experiment_headers_lower.keys())):
+return True
+
 uncommon_ctrl_headers = (set(control_headers_lower.keys()) -
  set(COMMON_SERVER_HEADERS))
 uncommon_exp_headers = (set(experiment_headers_lower.keys()) -



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Add support for Gzip Decoding in web connectivity test

[art]

commit f3cb68e06118ab7c20de9559404fae76715a5500
Author: Arturo Filastò 
Date:   Mon May 23 12:10:32 2016 +0200

Add support for Gzip Decoding in web connectivity test
---
 ooni/nettests/blocking/web_connectivity.py | 36 +-
 ooni/templates/httpt.py| 14 
 2 files changed, 40 insertions(+), 10 deletions(-)

diff --git a/ooni/nettests/blocking/web_connectivity.py 
b/ooni/nettests/blocking/web_connectivity.py
index 97cde60..f68b5aa 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -6,6 +6,7 @@ from urlparse import urlparse
 
 from ipaddr import IPv4Address, AddressValueError
 
+from twisted.web.client import GzipDecoder
 from twisted.internet import reactor
 from twisted.internet.protocol import Factory, Protocol
 from twisted.internet.endpoints import TCP4ClientEndpoint
@@ -72,6 +73,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 author = "Arturo Filastò"
 version = "0.1.0"
 
+contentDecoders = [('gzip', GzipDecoder)]
+
 usageOptions = UsageOptions
 
 inputFile = [
@@ -156,6 +159,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 self.report['dns_consistency'] = None
 self.report['body_length_match'] = None
 self.report['headers_match'] = None
+self.report['status_code_match'] = None
 
 self.report['accessible'] = None
 self.report['blocking'] = None
@@ -177,8 +181,9 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 'ips': []
 },
 'http_request': {
-'body_length': None,
-'failure': True,
+'body_length': -1,
+'failure': None,
+'status_code': -1,
 'headers': {}
 }
 }
@@ -270,6 +275,19 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 else:
 return False
 
+def compare_http_experiments(self, experiment_http_response):
+
+self.report['body_length_match'] = \
+self.compare_body_lengths(experiment_http_response)
+
+self.report['headers_match'] = \
+self.compare_headers(experiment_http_response)
+
+self.report['status_code_match'] = (
+experiment_http_response.code ==
+self.control['http_request']['status_code']
+)
+
 def compare_dns_experiments(self, experiment_dns_answers):
 if self.control['dns']['failure'] is not None and \
 self.control['dns']['failure'] == 
self.report['dns_experiment_failure']:
@@ -319,7 +337,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 def determine_blocking(self, experiment_http_response, 
experiment_dns_answers):
 blocking = False
 
-control_http_failure = 
self.report['control']['http_request']['failure']
+control_http_failure = self.control['http_request']['failure']
 if control_http_failure is not None:
 control_http_failure = control_http_failure.split(" ")[0]
 
@@ -328,10 +346,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 experiment_http_failure = experiment_http_failure.split(" ")[0]
 
 if (experiment_http_failure is None and control_http_failure is None):
-self.report['body_length_match'] = self.compare_body_lengths(
-experiment_http_response)
-self.report['headers_match'] = self.compare_headers(
-experiment_http_response)
+self.compare_http_experiments(experiment_http_response)
 
 dns_consistent = self.compare_dns_experiments(experiment_dns_answers)
 if dns_consistent is True:
@@ -341,7 +356,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 tcp_connect = self.compare_tcp_experiments()
 
 got_expected_web_page = (self.report['body_length_match'] or
- self.report['headers_match'])
+ self.report['headers_match']) and \
+self.report['status_code_match']
 
 if (dns_consistent == True and tcp_connect == False and
 experiment_http_failure is not None):
@@ -488,7 +504,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 
 for reason, urls in summary['blocked'].items():
 log.msg("")
-log.msg("URLS blocked due to {}".format(reason))
-log.msg(""+'-'*len(reason))
+log.msg("URLS possibly blocked due to {}".format(reason))
+log.msg("-"+'-'*len(reason))
 for url in urls:
 log.msg("* {}".format(url))
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index 7cbfd1d..ca0306b 100644
--- a/ooni/templates/httpt.py
+++ 

[tor-commits] [ooni-probe/master] Share code from ooni-backend via the ooni.common module

[art]

commit 3aa17317012aa0c85cff77d6942249f0bd68f8e6
Author: Arturo Filastò 
Date:   Sat May 28 15:45:05 2016 +0200

Share code from ooni-backend via the ooni.common module
---
 ooni/backend_client.py |   2 +-
 ooni/common/__init__.py|  10 +
 ooni/common/http_utils.py  |  55 +
 ooni/common/tcp_utils.py   |  10 +
 ooni/common/txextra.py | 202 +
 ooni/nettests/blocking/web_connectivity.py |  25 +--
 .../manipulation/http_header_field_manipulation.py |   2 +-
 ooni/templates/httpt.py|  42 +---
 ooni/templates/scapyt.py   |   8 +-
 ooni/tests/test_trueheaders.py |   2 +-
 ooni/utils/__init__.py |   8 -
 ooni/utils/net.py  |   8 -
 ooni/utils/socks.py|  25 +++
 ooni/utils/trueheaders.py  | 241 -
 14 files changed, 322 insertions(+), 318 deletions(-)

diff --git a/ooni/backend_client.py b/ooni/backend_client.py
index de01ceb..d4c463e 100644
--- a/ooni/backend_client.py
+++ b/ooni/backend_client.py
@@ -16,7 +16,7 @@ from ooni import errors as e
 from ooni.settings import config
 from ooni.utils import log, onion
 from ooni.utils.net import BodyReceiver, StringProducer, Downloader
-from ooni.utils.trueheaders import TrueHeadersSOCKS5Agent
+from ooni.utils.socks import TrueHeadersSOCKS5Agent
 
 
 class OONIBClient(object):
diff --git a/ooni/common/__init__.py b/ooni/common/__init__.py
new file mode 100644
index 000..7f6cf73
--- /dev/null
+++ b/ooni/common/__init__.py
@@ -0,0 +1,10 @@
+"""
+This modules contains functionality that is shared amongst ooni-probe and
+ooni-backend. If the code in here starts growing too much I think it would
+make sense to either:
+
+ * Make the code in here into it's own package that is imported by
+ ooni-probe and ooni-backend.
+
+ * Merge ooniprobe with oonibackend.
+"""
diff --git a/ooni/common/http_utils.py b/ooni/common/http_utils.py
new file mode 100644
index 000..6d636d5
--- /dev/null
+++ b/ooni/common/http_utils.py
@@ -0,0 +1,55 @@
+import re
+import codecs
+from base64 import b64encode
+
+META_CHARSET_REGEXP = 
re.compile(']*?charset\s*=[\s"\']*([^\s"\'/>]*)')
+
+def representBody(body):
+if not body:
+return body
+# XXX perhaps add support for decoding gzip in the future.
+body = body.replace('\0', '')
+decoded = False
+charsets = ['ascii', 'utf-8']
+
+# If we are able to detect the charset of body from the meta tag
+# try to decode using that one first
+charset = META_CHARSET_REGEXP.search(body, re.IGNORECASE)
+if charset:
+try:
+encoding = charset.group(1).lower()
+codecs.lookup(encoding)
+charsets.insert(0, encoding)
+except (LookupError, IndexError):
+# Skip invalid codecs and partial regexp match
+pass
+
+for encoding in charsets:
+try:
+body = unicode(body, encoding)
+decoded = True
+break
+except UnicodeDecodeError:
+pass
+if not decoded:
+body = {
+'data': b64encode(body),
+'format': 'base64'
+}
+return body
+
+TITLE_REGEXP = re.compile("(.*?)", re.IGNORECASE | re.DOTALL)
+
+def extractTitle(body):
+m = TITLE_REGEXP.search(body, re.IGNORECASE | re.DOTALL)
+if m:
+return unicode(m.group(1), errors='ignore')
+return ''
+
+REQUEST_HEADERS = {
+'User-Agent': ['Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, '
+   'like Gecko) Chrome/47.0.2526.106 Safari/537.36'],
+'Accept-Language': ['en-US;q=0.8,en;q=0.5'],
+'Accept': ['text/html,application/xhtml+xml,application/xml;q=0.9,'
+   '*/*;q=0.8']
+}
diff --git a/ooni/common/tcp_utils.py b/ooni/common/tcp_utils.py
new file mode 100644
index 000..7b7a8a4
--- /dev/null
+++ b/ooni/common/tcp_utils.py
@@ -0,0 +1,10 @@
+from twisted.internet.protocol import Factory, Protocol
+
+class TCPConnectProtocol(Protocol):
+def connectionMade(self):
+self.transport.loseConnection()
+
+class TCPConnectFactory(Factory):
+noisy = False
+def buildProtocol(self, addr):
+return TCPConnectProtocol()
diff --git a/ooni/common/txextra.py b/ooni/common/txextra.py
new file mode 100644
index 000..7a84592
--- /dev/null
+++ b/ooni/common/txextra.py
@@ -0,0 +1,202 @@
+import itertools
+from copy import copy
+
+from twisted.web.http_headers import Headers
+from twisted.web import error
+
+from twisted.web.client import BrowserLikeRedirectAgent
+from twisted.web._newclient import ResponseFailed
+from twisted.web._newclient import HTTPClientParser, ParseError
+from twisted.python.failure import Failure
+

[tor-commits] [ooni-probe/master] Add a timeout to the web_connectivity test to avoid the test timing out before it's done

[art]

commit a21ff3c777fb245096bc850532cf08ee0c4893d5
Author: Arturo Filastò 
Date:   Wed May 25 15:28:20 2016 +0200

Add a timeout to the web_connectivity test to avoid the test timing out 
before it's done
---
 ooni/nettests/blocking/web_connectivity.py | 18 ++
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/ooni/nettests/blocking/web_connectivity.py 
b/ooni/nettests/blocking/web_connectivity.py
index 7692d70..41f6c2a 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -50,6 +50,7 @@ class UsageOptions(usage.Options):
 ['dns-discovery', 'd', 'whoami.akamai.net', 'Specify the dns discovery 
test helper'],
 ['backend', 'b', None, 'The web_consistency backend test helper'],
 ['retries', 'r', 1, 'Number of retries for the HTTP request'],
+['timeout', 't', 240, 'Total timeout for this test'],
 ]
 
 
@@ -163,6 +164,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 except ValueError:
 self.localOptions['retries'] = 2
 
+self.timeout = int(self.localOptions['timeout'])
+
 self.report['retries'] = self.localOptions['retries']
 self.report['client_resolver'] = self.resolverIp
 self.report['dns_consistency'] = None
@@ -237,6 +240,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 result['status']['success'] = False
 result['status']['failure'] = failureToString(failure)
 self.report['tcp_connect'].append(result)
+
 return d
 
 @defer.inlineCallbacks
@@ -249,6 +253,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 
 @defer.inlineCallbacks
 def experiment_http_get_request(self):
+log.msg("* doing HTTP(s) request {}".format(self.input))
 retries = 0
 while True:
 try:
@@ -256,8 +261,10 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
   headers=REQUEST_HEADERS)
 break
 except:
-if self.localOptions['retries'] > retries:
+if retries > self.localOptions['retries']:
+log.debug("Finished all the allowed retries")
 raise
+log.debug("Re-running HTTP request")
 retries += 1
 
 defer.returnValue(result)
@@ -470,15 +477,18 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 experiment_http = self.experiment_http_get_request()
 @experiment_http.addErrback
 def http_experiment_err(failure):
-self.report['http_experiment_failure'] = failureToString(failure)
+failure_string = failureToString(failure)
+log.err("Failed to perform HTTP request %s" % failure_string)
+self.report['http_experiment_failure'] = failure_string
 
 experiment_http_response = yield experiment_http
 
 control_request = self.control_request(sockets)
 @control_request.addErrback
 def control_err(failure):
-log.err("Failed to perform control lookup")
-self.report['control_failure'] = failureToString(failure)
+failure_string = failureToString(failure)
+log.err("Failed to perform control lookup: %s" % failure_string)
+self.report['control_failure'] = failure_string
 
 yield control_request
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Increase the default timeout for the vanilla tor test to 300 seconds (#512)

[art]

commit a62a614de8839aee9341ad5663598cfcdd0bb668
Author: Arturo Filastò 
Date:   Mon May 23 14:57:18 2016 +0200

Increase the default timeout for the vanilla tor test to 300 seconds (#512)

This closes #508
---
 ooni/nettests/blocking/vanilla_tor.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ooni/nettests/blocking/vanilla_tor.py 
b/ooni/nettests/blocking/vanilla_tor.py
index 19c4805..7ed050f 100644
--- a/ooni/nettests/blocking/vanilla_tor.py
+++ b/ooni/nettests/blocking/vanilla_tor.py
@@ -18,7 +18,7 @@ class TorIsNotInstalled(Exception):
 
 class UsageOptions(usage.Options):
 optParameters = [
-['timeout', 't', 200,
+['timeout', 't', 300,
  'Specify the timeout after which to consider '
  'the Tor bootstrapping process to have failed.'], ]
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Clean up the output of ooniprobe

[art]

commit a1cf2789966b9550e3a044a0d3df7393328a7dee
Author: Arturo Filastò 
Date:   Tue May 17 17:36:47 2016 +0200

Clean up the output of ooniprobe

* Make OneShotFactory not be noisy

* Don't log an error when we are just connecting to a backend to check if 
it's
  up.
---
 ooni/backend_client.py |  2 +-
 ooni/utils/net.py  | 24 
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/ooni/backend_client.py b/ooni/backend_client.py
index ae376e4..3b551d4 100644
--- a/ooni/backend_client.py
+++ b/ooni/backend_client.py
@@ -143,7 +143,7 @@ class OONIBClient(object):
 except ValueError:
 raise e.get_error(None)
 if 'error' in response:
-log.err("Got this backend error message %s" % response)
+log.debug("Got this backend error message %s" % response)
 raise e.get_error(response['error'])
 return response
 
diff --git a/ooni/utils/net.py b/ooni/utils/net.py
index 1f87101..ad5454e 100644
--- a/ooni/utils/net.py
+++ b/ooni/utils/net.py
@@ -3,21 +3,21 @@ import socket
 from random import randint
 
 from zope.interface import implements
-from twisted.internet import protocol, defer
+from twisted.internet import defer
+from twisted.internet.protocol import Factory, Protocol
 from twisted.web.iweb import IBodyProducer
 
 from scapy.config import conf
 
 from ooni.errors import IfaceError
 
-try:
-from twisted.internet.endpoints import connectProtocol
-except ImportError:
-def connectProtocol(endpoint, protocol):
-class OneShotFactory(protocol.Factory):
-def buildProtocol(self, addr):
-return protocol
-return endpoint.connect(OneShotFactory())
+# This is our own connectProtocol to avoid noisy twisted cluttering our logs
+def connectProtocol(endpoint, protocol):
+class OneShotFactory(Factory):
+noisy = False
+def buildProtocol(self, addr):
+return protocol
+return endpoint.connect(OneShotFactory())
 
 # if sys.platform.system() == 'Windows':
 # import _winreg as winreg
@@ -68,7 +68,7 @@ class StringProducer(object):
 pass
 
 
-class BodyReceiver(protocol.Protocol):
+class BodyReceiver(Protocol):
 def __init__(self, finished, content_length=None, body_processor=None):
 self.finished = finished
 self.data = ""
@@ -92,7 +92,7 @@ class BodyReceiver(protocol.Protocol):
 self.finished.errback(exc)
 
 
-class Downloader(protocol.Protocol):
+class Downloader(Protocol):
 def __init__(self, download_path,
  finished, content_length=None):
 self.finished = finished
@@ -113,7 +113,7 @@ class Downloader(protocol.Protocol):
 self.finished.callback(None)
 
 
-class ConnectAndCloseProtocol(protocol.Protocol):
+class ConnectAndCloseProtocol(Protocol):
 def connectionMade(self):
 self.transport.loseConnection()
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Use .dev0 to indicate master branch to conform to PEP-440 (#507)

[art]

commit 2af0f7ebdc669926b21b3a1bb27712bc54d04ecb
Author: Arturo Filastò 
Date:   Fri May 20 17:55:28 2016 +0200

Use .dev0 to indicate master branch to conform to PEP-440 (#507)
---
 ooni/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ooni/__init__.py b/ooni/__init__.py
index 15e4278..e02b430 100644
--- a/ooni/__init__.py
+++ b/ooni/__init__.py
@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 
 __author__ = "Open Observatory of Network Interference"
-__version__ = "1.5.0-master"
+__version__ = "1.5.0.dev0"
 # This is the version number of resources to be downloaded
 # when a release is made it should be aligned to __version__
 __resources_version__ = "1.4.2"



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Compare CNAME responses as well

[art]

commit d7a7d0905862b7d75aaee20f32f351f1a66bb926
Author: Arturo Filastò 
Date:   Tue May 24 12:04:36 2016 +0200

Compare CNAME responses as well
---
 ooni/nettests/blocking/web_connectivity.py | 19 ++-
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/ooni/nettests/blocking/web_connectivity.py 
b/ooni/nettests/blocking/web_connectivity.py
index 3560cb0..e640a5f 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -181,7 +181,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 self.control = {
 'tcp_connect': {},
 'dns': {
-'ips': []
+'addrs': [],
+'failure': None,
 },
 'http_request': {
 'body_length': -1,
@@ -297,23 +298,23 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 self.report['dns_consistency'] = 'consistent'
 return True
 
-control_ips = set(self.control['dns']['ips'])
-experiment_ips = set(experiment_dns_answers)
+control_addrs = set(self.control['dns']['addrs'])
+experiment_addrs = set(experiment_dns_answers)
 
-if control_ips == experiment_ips:
+if control_addrs == experiment_addrs:
 return True
 
-for experiment_ip in experiment_ips:
-if is_public_ipv4_address(experiment_ip) is False:
+for experiment_addr in experiment_addrs:
+if is_public_ipv4_address(experiment_addr) is False:
 return False
 
-if len(control_ips.intersection(experiment_ips)) > 0:
+if len(control_addrs.intersection(experiment_addrs)) > 0:
 return True
 
 experiment_asns = set(map(lambda x: geoip.IPToLocation(x)['asn'],
-  experiment_ips))
+  experiment_addrs))
 control_asns = set(map(lambda x: geoip.IPToLocation(x)['asn'],
-   control_ips))
+   control_addrs))
 
 # Remove the instance of AS0 when we fail to find the ASN
 control_asns.discard('AS0')



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Merge pull request #511 from TheTorProject/fix/exit_ip

[art]

commit e2de248c9e1b778dabe345eb3d9a659ec6e88cc2
Merge: 07eef08 3ce772e
Author: anadahz 
Date:   Sat May 21 20:22:36 2016 +

Merge pull request #511 from TheTorProject/fix/exit_ip

Fix regression bug that lead to exit_ip and exit_name not being included

 ooni/templates/httpt.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Ensure the input key is properly set

[art]

commit c8df2379870e9210c4fe20541b5f6ca4f65e1cbb
Author: Arturo Filastò 
Date:   Mon May 23 13:12:36 2016 +0200

Ensure the input key is properly set

* Improve readability of the blockpage detection check
---
 ooni/nettests/blocking/web_connectivity.py | 8 +---
 ooni/tasks.py  | 5 ++---
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/ooni/nettests/blocking/web_connectivity.py 
b/ooni/nettests/blocking/web_connectivity.py
index f68b5aa..4f5f4e4 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -355,9 +355,11 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 self.report['dns_consistency'] = 'inconsistent'
 tcp_connect = self.compare_tcp_experiments()
 
-got_expected_web_page = (self.report['body_length_match'] or
- self.report['headers_match']) and \
-self.report['status_code_match']
+got_expected_web_page = (
+(self.report['body_length_match'] is True or
+ self.report['headers_match'] is True)
+and self.report['status_code_match'] is True
+)
 
 if (dns_consistent == True and tcp_connect == False and
 experiment_http_failure is not None):
diff --git a/ooni/tasks.py b/ooni/tasks.py
index 72e211f..925089b 100644
--- a/ooni/tasks.py
+++ b/ooni/tasks.py
@@ -113,10 +113,9 @@ class Measurement(TaskWithTimeout):
 if not hasattr(self.testInstance, '_start_time'):
 self.testInstance._start_time = time.time()
 
-if 'input' not in self.testInstance.report.keys():
-self.testInstance.report['input'] = test_input
-
 self.testInstance.setUp()
+if 'input' not in self.testInstance.report.keys():
+self.testInstance.report['input'] = self.testInstance.input
 
 self.netTestMethod = getattr(self.testInstance, test_method)
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Refactor code for setting of tor port

[art]

commit 1e3954bb26b2ac94b968d3e60cbf7ef75bb4c9b9
Author: Arturo Filastò 
Date:   Fri May 13 15:33:00 2016 +0200

Refactor code for setting of tor port
---
 ooni/director.py | 22 ++
 1 file changed, 6 insertions(+), 16 deletions(-)

diff --git a/ooni/director.py b/ooni/director.py
index 82d0e85..e6f864e 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -345,11 +345,13 @@ class Director(object):
 log.msg("%d%%: %s" % (prog, summary))
 
 tor_config = TorConfig()
-if config.tor.control_port:
-tor_config.ControlPort = config.tor.control_port
+if config.tor.control_port is None:
+config.tor.control_port = int(randomFreePort())
+if config.tor.socks_port is None:
+config.tor.socks_port = int(randomFreePort())
 
-if config.tor.socks_port:
-tor_config.SocksPort = config.tor.socks_port
+tor_config.ControlPort = config.tor.control_port
+tor_config.SocksPort = config.tor.socks_port
 
 if config.tor.data_dir:
 data_dir = os.path.expanduser(config.tor.data_dir)
@@ -384,18 +386,6 @@ class Director(object):
 tor_config.User = pwd.getpwuid(os.geteuid()).pw_name
 
 tor_config.save()
-
-if not hasattr(tor_config, 'ControlPort'):
-control_port = int(randomFreePort())
-tor_config.ControlPort = control_port
-config.tor.control_port = control_port
-
-if not hasattr(tor_config, 'SocksPort'):
-socks_port = int(randomFreePort())
-tor_config.SocksPort = socks_port
-config.tor.socks_port = socks_port
-
-tor_config.save()
 log.debug("Setting control port as %s" % tor_config.ControlPort)
 log.debug("Setting SOCKS port as %s" % tor_config.SocksPort)
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Fix regression bug that lead to exit_ip and exit_name not being included

[art]

commit 3ce772e21ca395993d3ad2878cc394a768f5be79
Author: Arturo Filastò 
Date:   Sat May 21 19:33:42 2016 +0200

Fix regression bug that lead to exit_ip and exit_name not being included

This closes #509
---
 ooni/templates/httpt.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index e608f8f..fe283a7 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -30,7 +30,7 @@ class StreamListener(StreamListenerMixin):
 def stream_succeeded(self, stream):
 host=self.request['url'].split('/')[2]
 try:
-if stream.target_host == host and len(self.request['tor']) == 1:
+if stream.target_host == host and self.request['tor']['exit_ip'] 
is None:
 self.request['tor']['exit_ip'] = stream.circuit.path[-1].ip
 self.request['tor']['exit_name'] = stream.circuit.path[-1].name
 config.tor_state.stream_listeners.remove(self)



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Change test for setupCollector() function

[art]

commit 8fd571f03319462babc1ca925cb147498d28f138
Author: Arturo Filastò 
Date:   Wed May 18 16:10:07 2016 +0200

Change test for setupCollector() function
---
 ooni/tests/test_oonicli.py | 10 --
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py
index 24877e8..4a58736 100644
--- a/ooni/tests/test_oonicli.py
+++ b/ooni/tests/test_oonicli.py
@@ -221,5 +221,11 @@ class TestOoniCli(ConfigTestCase):
 global_options = {
 'collector': collector1
 }
-collector_address = setupCollector(global_options, collector2)
-self.assertEqual(collector_address, collector1)
+try:
+collector_client = setupCollector(global_options, collector2)
+self.assertEqual(collector_client.settings['address'], collector1)
+self.assertEqual(collector_client.settings['type'], 'https')
+except errors.CollectorUnsupported:
+# Older versions of twisted will raise this. We could be more
+# strict and do a check for older twisted versions in here.
+pass



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Consider valid onion addresses also when they don't use the httpo:// schema

[art]

commit c32bd88b5b99ea7d3fc62c9088d3ac17770ba7df
Author: Arturo Filastò 
Date:   Tue May 24 11:18:10 2016 +0200

Consider valid onion addresses also when they don't use the httpo:// schema

Based on feedback from @willscott

* Fix invalid onion address in unittests
---
 ooni/backend_client.py   | 16 ++--
 ooni/deck.py |  4 ++--
 ooni/tests/mocks.py  |  6 +++---
 ooni/tests/test_deck.py  |  2 +-
 ooni/tests/test_onion.py | 16 
 ooni/utils/onion.py  |  8 
 6 files changed, 40 insertions(+), 12 deletions(-)

diff --git a/ooni/backend_client.py b/ooni/backend_client.py
index 3b551d4..de01ceb 100644
--- a/ooni/backend_client.py
+++ b/ooni/backend_client.py
@@ -14,7 +14,7 @@ _twisted_14_0_2_version = Version('twisted', 14, 0, 2)
 
 from ooni import errors as e
 from ooni.settings import config
-from ooni.utils import log
+from ooni.utils import log, onion
 from ooni.utils.net import BodyReceiver, StringProducer, Downloader
 from ooni.utils.trueheaders import TrueHeadersSOCKS5Agent
 
@@ -42,10 +42,10 @@ class OONIBClient(object):
 def _guessBackendType(self):
 if self.base_address is None:
 raise e.InvalidAddress
-if self.base_address.startswith('https://'):
-self.backend_type = 'https'
-elif self.base_address.startswith('httpo://'):
+if onion.is_onion_address(self.base_address):
 self.backend_type = 'onion'
+elif self.base_address.startswith('https://'):
+self.backend_type = 'https'
 elif self.base_address.startswith('http://'):
 self.backend_type = 'http'
 else:
@@ -54,10 +54,14 @@ class OONIBClient(object):
 def _setupBaseAddress(self):
 parsed_address = urlparse(self.base_address)
 if self.backend_type == 'onion':
-if not parsed_address.netloc.endswith(".onion"):
+if not onion.is_onion_address(self.base_address):
 log.err("Invalid onion address.")
 raise e.InvalidAddress(self.base_address)
-self.base_address = ("http://%s; % parsed_address.netloc)
+if parsed_address.scheme in ('http', 'httpo'):
+self.base_address = ("http://%s; % parsed_address.netloc)
+else:
+self.base_address = ("%s://%s" % (parsed_address.scheme,
+  parsed_address.netloc))
 elif self.backend_type == 'http':
 self.base_address = ("http://%s; % parsed_address.netloc)
 elif self.backend_type in ('https', 'cloudfront'):
diff --git a/ooni/deck.py b/ooni/deck.py
index 7a3b697..383845c 100644
--- a/ooni/deck.py
+++ b/ooni/deck.py
@@ -4,7 +4,7 @@ from ooni.backend_client import CollectorClient, BouncerClient
 from ooni.backend_client import WebConnectivityClient
 from ooni.nettest import NetTestLoader
 from ooni.settings import config
-from ooni.utils import log
+from ooni.utils import log, onion
 from ooni import errors as e
 
 from twisted.python.filepath import FilePath
@@ -175,7 +175,7 @@ class Deck(InputFile):
 https_addresses = []
 plaintext_addresses = []
 
-if priority_address.startswith('httpo://'):
+if onion.is_onion_address(priority_address):
 priority_address = {
 'address': priority_address,
 'type': 'onion'
diff --git a/ooni/tests/mocks.py b/ooni/tests/mocks.py
index f3f852f..db7b154 100644
--- a/ooni/tests/mocks.py
+++ b/ooni/tests/mocks.py
@@ -197,13 +197,13 @@ class MockBouncerClient(object):
 ret = {
 'default': {
 'address': '127.0.0.1',
-'collector': 'httpo://thirteenchars1234.onion'
+'collector': 'httpo://thirteenchars123.onion'
 }
 }
 for required_test_helper in required_test_helpers:
 ret[required_test_helper] = {
 'address': '127.0.0.1',
-'collector': 'httpo://thirteenchars1234.onion'
+'collector': 'httpo://thirteenchars123.onion'
 }
 return defer.succeed(ret)
 
@@ -221,7 +221,7 @@ class MockBouncerClient(object):
 'name': net_test['name'],
 'version': net_test['version'],
 'input-hashes': net_test['input-hashes'],
-'collector': 'httpo://thirteenchars1234.onion',
+'collector': 'httpo://thirteenchars123.onion',
 'test-helpers': test_helpers
 })
 return defer.succeed(ret)
diff --git a/ooni/tests/test_deck.py b/ooni/tests/test_deck.py
index 8d415f0..fa9b0e8 100644
--- a/ooni/tests/test_deck.py
+++ b/ooni/tests/test_deck.py
@@ -160,7 +160,7 @@ class TestDeck(BaseTestCase):
 yield deck.lookupCollectorAndTestHelpers()
 
 self.assertEqual(deck.netTestLoaders[0].collector.settings['address'],
- 

[tor-commits] [ooni-probe/master] Use settings.Config.ooni_home as the data directory (to avoid broken paths when using sudo or setting environment vars)

[art]

commit edcb8379aa78d2e9fdeb039aa7e6fd82170fa2dd
Author: Arturo Filastò 
Date:   Mon Apr 18 18:46:33 2016 +0200

Use settings.Config.ooni_home as the data directory (to avoid broken paths 
when using sudo or setting environment vars)
---
 ooni/settings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ooni/settings.py b/ooni/settings.py
index 6e0f9a5..ffbf68e 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -58,7 +58,7 @@ class OConfig(object):
 @property
 def data_directory_candidates(self):
 dirs = [
-os.path.join(expanduser('~'+self.current_user), '.ooni'),
+self.ooni_home,
 self.var_lib_path,
 self.usr_share_path,
 os.path.join(get_ooni_root(), '..', 'data'),



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Fix some issues with the rebase

[art]

commit 8a6e1a50fc5aa6fc9726b111c9edc9f44008f921
Author: Arturo Filastò 
Date:   Thu May 5 19:30:29 2016 +0200

Fix some issues with the rebase
---
 ooni/nettest.py| 2 --
 ooni/reporter.py   | 6 +++---
 ooni/tests/test_nettest.py | 9 ++---
 3 files changed, 5 insertions(+), 12 deletions(-)

diff --git a/ooni/nettest.py b/ooni/nettest.py
index 67d3e27..8b9556e 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -572,8 +572,6 @@ class NetTest(object):
 test_class.inputs = yield defer.maybeDeferred(
 test_class().getInputProcessor
 )
-if not test_class.inputs:
-test_class.inputs = [None]
 
 # Run the setupClass method
 yield defer.maybeDeferred(
diff --git a/ooni/reporter.py b/ooni/reporter.py
index aa6f5ee..6103c1e 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -646,12 +646,12 @@ class Report(object):
 created callback of the reporter whose report got created.
 """
 if self.collector_address:
-self.oonib_reporter = OONIBReporter(self.net_test_details,
+self.oonib_reporter = OONIBReporter(self.test_details,
 self.collector_address)
-self.net_test_details['report_id'] = yield 
self.open_oonib_reporter()
+self.test_details['report_id'] = yield self.open_oonib_reporter()
 
 if not self.no_yamloo:
-self.yaml_reporter = YAMLReporter(self.net_test_details,
+self.yaml_reporter = YAMLReporter(self.test_details,
   
report_filename=self.report_filename)
 self.report_filename = self.yaml_reporter.report_path
 if not self.oonib_reporter:
diff --git a/ooni/tests/test_nettest.py b/ooni/tests/test_nettest.py
index 660fd3e..da4969f 100644
--- a/ooni/tests/test_nettest.py
+++ b/ooni/tests/test_nettest.py
@@ -261,13 +261,8 @@ class TestNetTest(unittest.TestCase):
 nt = NetTest(ntl.getTestCases(), ntl.getTestDetails(), None)
 yield nt.initialize()
 
-# XXX: if you use the same test_class twice you will have consumed all
-# of its inputs!
-tested = set([])
-for test_class, test_method in ntl.getTestCases():
-if test_class not in tested:
-tested.update([test_class])
-self.assertEqual(len(list(test_class.inputs)), 10)
+for test_class, test_methods in nt.testCases:
+self.assertEqual(len(list(test_class.inputs)), 10)
 
 def test_setup_local_options_in_test_cases(self):
 ntl = NetTestLoader(dummyArgs)



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Add the redirected responses in inverted order

[art]

commit 32c463b766476a0db697d8c72264ed5683563fd2
Author: Arturo Filastò 
Date:   Sun May 8 19:08:52 2016 +0200

Add the redirected responses in inverted order

This means the first item in the list will be the final response, while the
last will be the first response.
---
 ooni/templates/httpt.py | 11 ++-
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index 6b4c4b9..c58daf2 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -162,11 +162,6 @@ class HTTPTest(NetTestCase):
 
 failure (instance): An instance of 
:class:twisted.internet.failure.Failure
 """
-if response and response.previousResponse:
-self.addToReport(request, response.previousResponse,
- response_body=None,
- failure_string=None)
-
 log.debug("Adding %s to report" % request)
 request_headers = TrueHeaders(request['headers'])
 session = {
@@ -199,6 +194,12 @@ class HTTPTest(NetTestCase):
 
 self.report['requests'].append(session)
 
+if response and response.previousResponse:
+self.addToReport(request, response.previousResponse,
+ response_body=None,
+ failure_string=None)
+
+
 def _processResponseBody(self, response_body, request, response, 
body_processor):
 log.debug("Processing response body")
 HTTPTest.addToReport(self, request, response, response_body)



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Patch twisted HTTPClientParser to accept HTTP response lines with a missing reason

[art]

commit c50f50cb0984cefabc82bd4f559062dbc893736c
Author: Arturo Filastò 
Date:   Sun May 8 19:09:46 2016 +0200

Patch twisted HTTPClientParser to accept HTTP response lines with a missing 
reason
---
 ooni/utils/trueheaders.py | 20 
 1 file changed, 20 insertions(+)

diff --git a/ooni/utils/trueheaders.py b/ooni/utils/trueheaders.py
index a5b2fe3..5f2b399 100644
--- a/ooni/utils/trueheaders.py
+++ b/ooni/utils/trueheaders.py
@@ -115,6 +115,26 @@ class HTTPClientParser(_newclient.HTTPClientParser):
 headers.addRawHeader(name, value)
 
 
+def statusReceived(self, status):
+parts = status.split(b' ', 2)
+if len(parts) != 3:
+# Here we add the extra missing part.
+parts.append("XXX")
+
+try:
+statusCode = int(parts[1])
+except ValueError:
+raise _newclient.ParseError(u"non-integer status code", status)
+
+self.response = _newclient.Response._construct(
+self.parseVersion(parts[0]),
+statusCode,
+parts[2],
+self.headers,
+self.transport,
+self.request)
+
+
 class HTTP11ClientProtocol(_newclient.HTTP11ClientProtocol):
 def request(self, request):
 if self._state != 'QUIESCENT':



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Take also into consideration if the HTTP headers match in detecting blockpages.

[art]

commit 958850898457e78e7091dd2aa5193e9956d26a8f
Author: Arturo Filastò 
Date:   Mon May 9 15:56:35 2016 +0200

Take also into consideration if the HTTP headers match in detecting 
blockpages.

This significantly brings down the false positive ratio. Though there are 
still
some instances of sites being mis-reported as blocked.

Examples of these are sites that have different web servers for sites in the
various languages.
---
 ooni/nettests/blocking/web_connectivity.py | 25 +
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/ooni/nettests/blocking/web_connectivity.py 
b/ooni/nettests/blocking/web_connectivity.py
index 08cae57..eb05835 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -347,6 +347,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 self.report['dns_consistency'] = 'inconsistent'
 tcp_connect = self.compare_tcp_experiments()
 
+got_expected_web_page = (self.report['body_length_match'] or
+ self.report['headers_match'])
 
 if (dns_consistent == True and tcp_connect == False and
 experiment_http_failure is not None):
@@ -354,23 +356,30 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 
 # XXX we may want to have different codes for these two types of
 # blocking
-elif (dns_consistent == True and tcp_connect == True and
-  self.report['body_length_match'] == False):
-blocking = 'http'
-elif (dns_consistent == True and tcp_connect == True and
-experiment_http_failure is not None and
-control_http_failure is None):
+elif (dns_consistent == True and
+  tcp_connect == True and
+  got_expected_web_page == False):
 blocking = 'http'
 
+elif (dns_consistent == True and
+  tcp_connect == True and
+  experiment_http_failure is not None and
+  control_http_failure is None):
+if experiment_http_failure == 'dns_lookup_error':
+blocking = 'dns'
+else:
+blocking = 'http'
+
 elif (dns_consistent == False and
-  (experiment_http_failure is not None or
-   self.report['body_length_match'] == False)):
+  (got_expected_web_page == False or
+experiment_http_failure is not None)):
 blocking = 'dns'
 
 # This happens when the DNS resolution is injected, but the domain
 # doesn't have a valid record anymore or it resolves to an address
 # that is only accessible from within the country/network of the probe.
 elif (dns_consistent == False and
+  got_expected_web_page == False and
   (self.control['dns']['failure'] is not None or
control_http_failure != experiment_http_failure)):
 blocking = 'dns'



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Ignore empty charset values

[art]

commit d80a68098dc4fd13a4963e26d6c4e5ed2ad31b43
Author: Arturo Filastò 
Date:   Sun May 8 17:55:46 2016 +0200

Ignore empty charset values
---
 ooni/templates/httpt.py  | 2 +-
 ooni/tests/test_templates.py | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index 2a17f5b..51fba1a 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -17,7 +17,7 @@ from ooni.utils.net import StringProducer, userAgents
 from ooni.utils.trueheaders import TrueHeaders
 from ooni.errors import handleAllFailures
 
-META_CHARSET_REGEXP = 
re.compile(']*?charset\s*=[\s"\']*([^\s"\'/>]*)')
+META_CHARSET_REGEXP = 
re.compile(']*?charset\s*=[\s"\']*([^\s"\'/>]+)')
 
 class InvalidSocksProxyOption(Exception):
 pass
diff --git a/ooni/tests/test_templates.py b/ooni/tests/test_templates.py
index 5b2c77a..e8fe636 100644
--- a/ooni/tests/test_templates.py
+++ b/ooni/tests/test_templates.py
@@ -54,8 +54,10 @@ class TestHTTPT(unittest.TestCase):
 Foo
 """
 with_charset_html = no_charset_html + '\n'
+with_empty_charset = no_charset_html + '\n'
 self.assertEqual(httpt.META_CHARSET_REGEXP.search(no_charset_html), 
None)
 
self.assertEqual(httpt.META_CHARSET_REGEXP.search(with_charset_html).group(1), 
'iso-8859-1')
+self.assertEqual(httpt.META_CHARSET_REGEXP.search(with_empty_charset), 
None)
 
 class TestDNST(unittest.TestCase):
 def setUp(self):



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Improve detection logic for web_connectivity

[art]

commit 4dd1542306a6cb3615cd5844728a689a4be7c998
Author: Arturo Filastò 
Date:   Tue May 3 15:23:18 2016 +0200

Improve detection logic for web_connectivity

* Fix printing of exception log
---
 bin/oonireport |  2 +-
 ooni/nettests/blocking/web_connectivity.py | 22 +++---
 ooni/utils/log.py  |  2 +-
 3 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/bin/oonireport b/bin/oonireport
index 6a79f4c..9f59683 100755
--- a/bin/oonireport
+++ b/bin/oonireport
@@ -14,8 +14,8 @@ def failed(failure):
 r = failure.trap(exceptions.SystemExit,
  Exception)
 if r != exceptions.SystemExit:
-log.err("Failed to run oonireport")
 log.exception(failure)
+log.err("Failed to run oonireport")
 exitCode = 127
 else:
 exitCode = failure.value.code
diff --git a/ooni/nettests/blocking/web_connectivity.py 
b/ooni/nettests/blocking/web_connectivity.py
index 69fd002..08cae57 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -326,8 +326,15 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 def determine_blocking(self, experiment_http_response, 
experiment_dns_answers):
 blocking = False
 
-if (self.report['http_experiment_failure'] is None and
-self.report['control']['http_request']['failure'] is None):
+control_http_failure = 
self.report['control']['http_request']['failure']
+if control_http_failure is not None:
+control_http_failure = control_http_failure.split(" ")[0]
+
+experiment_http_failure = self.report['http_experiment_failure']
+if experiment_http_failure is not None:
+experiment_http_failure = experiment_http_failure.split(" ")[0]
+
+if (experiment_http_failure is None and control_http_failure is None):
 self.report['body_length_match'] = self.compare_body_lengths(
 experiment_http_response)
 self.report['headers_match'] = self.compare_headers(
@@ -340,8 +347,9 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 self.report['dns_consistency'] = 'inconsistent'
 tcp_connect = self.compare_tcp_experiments()
 
+
 if (dns_consistent == True and tcp_connect == False and
-self.report['http_experiment_failure'] is not None):
+experiment_http_failure is not None):
 blocking = 'tcp_ip'
 
 # XXX we may want to have different codes for these two types of
@@ -350,12 +358,12 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
   self.report['body_length_match'] == False):
 blocking = 'http'
 elif (dns_consistent == True and tcp_connect == True and
-self.report['http_experiment_failure'] is not None and
-self.report['control']['http_request']['failure'] != 
self.report['http_experiment_failure']):
+experiment_http_failure is not None and
+control_http_failure is None):
 blocking = 'http'
 
 elif (dns_consistent == False and
-  (self.report['http_experiment_failure'] is not None or
+  (experiment_http_failure is not None or
self.report['body_length_match'] == False)):
 blocking = 'dns'
 
@@ -364,7 +372,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 # that is only accessible from within the country/network of the probe.
 elif (dns_consistent == False and
   (self.control['dns']['failure'] is not None or
-   self.control['http_request']['failure'] is not None)):
+   control_http_failure != experiment_http_failure)):
 blocking = 'dns'
 
 return blocking
diff --git a/ooni/utils/log.py b/ooni/utils/log.py
index f36c0ff..fd80594 100644
--- a/ooni/utils/log.py
+++ b/ooni/utils/log.py
@@ -111,7 +111,7 @@ def exception(error):
 or it can be a twisted.python.failure.Failure instance.
 """
 if isinstance(error, Failure):
-error.printTraceback()
+error.printTraceback(sys.stdout)
 else:
 exc_type, exc_value, exc_traceback = sys.exc_info()
 traceback.print_exception(exc_type, exc_value, exc_traceback)



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Make the test-helper-alternate key work when speaking to backend

[art]

commit 2b238683d01d2d974a3743a226dd30f02371faf0
Author: Arturo Filastò 
Date:   Fri May 13 15:31:58 2016 +0200

Make the test-helper-alternate key work when speaking to backend

* Rename advanced option insecure_collector to insecure_backend

* Refactor isReachable() to only return booleans

* Bugfixing
---
 data/ooniprobe.conf.sample |  2 +-
 ooni/backend_client.py | 21 -
 ooni/deck.py   | 70 +-
 ooni/errors.py | 11 +
 ooni/nettests/blocking/web_connectivity.py |  1 +
 ooni/oonicli.py|  4 +-
 6 files changed, 45 insertions(+), 64 deletions(-)

diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample
index edc3f86..72a8e06 100644
--- a/data/ooniprobe.conf.sample
+++ b/data/ooniprobe.conf.sample
@@ -48,7 +48,7 @@ advanced:
 report_log_file: null
 inputs_dir: null
 decks_dir: null
-insecure_collector: false
+insecure_backend: false
 tor:
 #socks_port: 8801
 #control_port: 8802
diff --git a/ooni/backend_client.py b/ooni/backend_client.py
index c7de7f0..ae376e4 100644
--- a/ooni/backend_client.py
+++ b/ooni/backend_client.py
@@ -23,7 +23,7 @@ class OONIBClient(object):
 def __init__(self, address=None, settings={}):
 self.base_headers = {}
 self.backend_type = settings.get('type', None)
-self.base_address = settings.get('address', address).encode('ascii')
+self.base_address = settings.get('address', address)
 
 if self.backend_type is None:
 self._guessBackendType()
@@ -62,6 +62,7 @@ class OONIBClient(object):
 self.base_address = ("http://%s; % parsed_address.netloc)
 elif self.backend_type in ('https', 'cloudfront'):
 self.base_address = ("https://%s; % parsed_address.netloc)
+self.base_address = self.base_address.encode('ascii')
 
 def isSupported(self):
 if self.backend_type in ("https", "cloudfront"):
@@ -70,10 +71,10 @@ class OONIBClient(object):
 "twisted > 14.0.2.")
 return False
 elif self.backend_type == "http":
-if config.advanced.insecure_collector is not True:
+if config.advanced.insecure_backend is not True:
 log.err("Plaintext backends are not supported. To "
 "enable at your own risk set "
-"advanced->insecure_collector to true")
+"advanced->insecure_backend to true")
 return False
 elif self.backend_type == "onion":
 # XXX add an extra check to ensure tor is running
@@ -159,7 +160,7 @@ class OONIBClient(object):
 
 class BouncerClient(OONIBClient):
 def isReachable(self):
-pass
+return defer.succeed(True)
 
 @defer.inlineCallbacks
 def lookupTestCollector(self, net_tests):
@@ -197,14 +198,14 @@ class CollectorClient(OONIBClient):
 def cb(_):
 # We should never be getting an acceptable response for a
 # request to an invalid path.
-raise e.CollectorUnreachable
+return False
 
 @d.addErrback
 def err(failure):
 failure.trap(Error)
 if failure.value.status == '404':
 return True
-raise e.CollectorUnreachable
+return False
 
 return d
 
@@ -346,13 +347,13 @@ class WebConnectivityClient(OONIBClient):
 
 @d.addCallback
 def cb(result):
-if result.get("status", None) is not "ok":
-raise e.TestHelperUnreachable
+if result.get("status", None) != "ok":
+return False
 return True
 
 @d.addErrback
 def err(_):
-raise e.TestHelperUnreachable
+return False
 
 return d
 
@@ -361,4 +362,4 @@ class WebConnectivityClient(OONIBClient):
 'http_request': http_request,
 'tcp_connect': tcp_connect
 }
-self.queryBackend('POST', '/', query=request)
+return self.queryBackend('POST', '/', query=request)
diff --git a/ooni/deck.py b/ooni/deck.py
index 24c7904..007097e 100644
--- a/ooni/deck.py
+++ b/ooni/deck.py
@@ -184,11 +184,10 @@ class Deck(InputFile):
 'type': 'https'
 }
 elif priority_address.startswith('http://'):
-if config.advanced.insecure_collector is True:
-priority_address = {
-'address': priority_address,
-'type': 'http'
-}
+priority_address = {
+'address': priority_address,
+'type': 'http'
+}
 else:
 raise e.InvalidOONIBCollectorAddress
 
@@ -200,13 +199,13 @@ class Deck(InputFile):
 cloudfront_addresses += 

[tor-commits] [ooni-probe/master] Start adding support for HTTPS, Cloudfronted test helpers and collectors

[art]

commit d17873211da4bd6ec3c0d449ea1d62c2216d1996
Author: Arturo Filastò 
Date:   Wed May 11 15:11:13 2016 +0200

Start adding support for HTTPS, Cloudfronted test helpers and collectors

* Add routines to verify which collectors and test helpers are reachable

* Merciless refactoring of the reporting logic

* Remove dumb logic
---
 ooni/backend_client.py | 364 +
 ooni/deck.py   | 164 +++--
 ooni/director.py   |  24 +-
 ooni/errors.py |  49 +++-
 ooni/nettests/blocking/web_connectivity.py |  36 ++-
 ooni/oonibclient.py| 232 --
 ooni/oonicli.py|  35 +--
 ooni/report/tool.py|   4 +-
 ooni/reporter.py   | 220 +
 ooni/tests/mocks.py|  12 +-
 ooni/tests/test_deck.py|   9 +-
 ooni/tests/test_oonibclient.py |  47 ++--
 ooni/tests/test_reporter.py|  32 ++-
 ooni/tests/test_utils.py   |  34 +--
 ooni/utils/__init__.py |  56 ++---
 15 files changed, 729 insertions(+), 589 deletions(-)

diff --git a/ooni/backend_client.py b/ooni/backend_client.py
new file mode 100644
index 000..c7de7f0
--- /dev/null
+++ b/ooni/backend_client.py
@@ -0,0 +1,364 @@
+import os
+import json
+
+from urlparse import urljoin, urlparse
+
+from twisted.web.error import Error
+from twisted.web.client import Agent, Headers
+from twisted.internet import defer, reactor
+from twisted.internet.endpoints import TCP4ClientEndpoint
+
+from twisted.python.versions import Version
+from twisted import version as _twisted_version
+_twisted_14_0_2_version = Version('twisted', 14, 0, 2)
+
+from ooni import errors as e
+from ooni.settings import config
+from ooni.utils import log
+from ooni.utils.net import BodyReceiver, StringProducer, Downloader
+from ooni.utils.trueheaders import TrueHeadersSOCKS5Agent
+
+
+class OONIBClient(object):
+def __init__(self, address=None, settings={}):
+self.base_headers = {}
+self.backend_type = settings.get('type', None)
+self.base_address = settings.get('address', address).encode('ascii')
+
+if self.backend_type is None:
+self._guessBackendType()
+self.backend_type = self.backend_type.encode('ascii')
+
+if self.backend_type == 'cloudfront':
+self.base_headers['Host'] = settings['front'].encode('ascii')
+
+self._setupBaseAddress()
+self.settings = {
+'type': self.backend_type,
+'address': self.base_address,
+'front': settings.get('front', '').encode('ascii')
+}
+
+def _guessBackendType(self):
+if self.base_address is None:
+raise e.InvalidAddress
+if self.base_address.startswith('https://'):
+self.backend_type = 'https'
+elif self.base_address.startswith('httpo://'):
+self.backend_type = 'onion'
+elif self.base_address.startswith('http://'):
+self.backend_type = 'http'
+else:
+raise e.InvalidAddress
+
+def _setupBaseAddress(self):
+parsed_address = urlparse(self.base_address)
+if self.backend_type == 'onion':
+if not parsed_address.netloc.endswith(".onion"):
+log.err("Invalid onion address.")
+raise e.InvalidAddress(self.base_address)
+self.base_address = ("http://%s; % parsed_address.netloc)
+elif self.backend_type == 'http':
+self.base_address = ("http://%s; % parsed_address.netloc)
+elif self.backend_type in ('https', 'cloudfront'):
+self.base_address = ("https://%s; % parsed_address.netloc)
+
+def isSupported(self):
+if self.backend_type in ("https", "cloudfront"):
+if _twisted_version < _twisted_14_0_2_version:
+log.err("HTTPS and cloudfronted backends require "
+"twisted > 14.0.2.")
+return False
+elif self.backend_type == "http":
+if config.advanced.insecure_collector is not True:
+log.err("Plaintext backends are not supported. To "
+"enable at your own risk set "
+"advanced->insecure_collector to true")
+return False
+elif self.backend_type == "onion":
+# XXX add an extra check to ensure tor is running
+if not config.tor_state and config.tor.socks_port is None:
+return False
+return True
+
+def isReachable(self):
+raise NotImplemented
+
+def _request(self, method, urn, genReceiver, bodyProducer=None, retries=3):
+if self.backend_type == 'onion':
+agent = 

[tor-commits] [ooni-probe/master] Improvements to the output of the web_connectivity test

[art]

commit 6f0154b33f3528e3fdd78ca0f312e174d89bae33
Author: Arturo Filastò 
Date:   Mon May 2 17:53:58 2016 +0200

Improvements to the output of the web_connectivity test
---
 ooni/nettests/blocking/web_connectivity.py | 82 ++
 1 file changed, 71 insertions(+), 11 deletions(-)

diff --git a/ooni/nettests/blocking/web_connectivity.py 
b/ooni/nettests/blocking/web_connectivity.py
index 28ee942..69fd002 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -1,5 +1,6 @@
 # -*- encoding: utf-8 -*-
 
+import csv
 import json
 from urlparse import urlparse
 
@@ -102,6 +103,43 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 log.exception(exc)
 log.err("Failed to lookup the resolver IP address")
 
+
+def inputProcessor(self, filename):
+"""
+This is a specialised inputProcessor that also supports taking as
+input a csv file.
+"""
+def csv_generator(fh):
+for row in csv.reader(fh):
+yield row[0]
+
+def simple_file_generator(fh):
+for line in fh:
+l = line.strip()
+# Skip empty lines
+if not l:
+continue
+# Skip comment lines
+elif l.startswith('#'):
+continue
+yield l
+
+try:
+fh = open(filename)
+line = fh.readline()
+# Detect the line of the citizenlab input file
+if line.startswith("url,"):
+generator = csv_generator(fh)
+else:
+fh.seek(0)
+generator = simple_file_generator(fh)
+for i in generator:
+if not i.startswith("http"):
+i = "http://{}/".format(i)
+yield i
+finally:
+fh.close()
+
 def setUp(self):
 """
 Check for inputs.
@@ -143,10 +181,11 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 }
 
 def experiment_dns_query(self):
-log.msg("Doing DNS query for {}".format(self.hostname))
+log.msg("* doing DNS query for {}".format(self.hostname))
 return self.performALookup(self.hostname)
 
 def tcp_connect(self, socket):
+log.msg("* connecting to {}".format(socket))
 ip_address, port = socket.split(":")
 port = int(port)
 result = {
@@ -212,7 +251,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 try:
 control_headers_lower[header_name.lower()]
 except KeyError:
-log.msg("Did not find the key {}".format(header_name))
+log.debug("Did not find the key {}".format(header_name))
 return False
 count += 1
 
@@ -262,6 +301,10 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 control_asns = set(map(lambda x: geoip.IPToLocation(x)['asn'],
control_ips))
 
+# Remove the instance of AS0 when we fail to find the ASN
+control_asns.discard('AS0')
+experiment_asns.discard('AS0')
+
 if len(control_asns.intersection(experiment_asns)) > 0:
 return True
 
@@ -316,11 +359,21 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
self.report['body_length_match'] == False)):
 blocking = 'dns'
 
+# This happens when the DNS resolution is injected, but the domain
+# doesn't have a valid record anymore or it resolves to an address
+# that is only accessible from within the country/network of the probe.
+elif (dns_consistent == False and
+  (self.control['dns']['failure'] is not None or
+   self.control['http_request']['failure'] is not None)):
+blocking = 'dns'
+
 return blocking
 
 
 @defer.inlineCallbacks
 def test_web_connectivity(self):
+log.msg("")
+log.msg("Starting test for {}".format(self.input))
 experiment_dns = self.experiment_dns_query()
 
 @experiment_dns.addErrback
@@ -385,15 +438,20 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 else:
 log.msg("* Is NOT accessible")
 self.report['accessible'] = False
+log.msg("")
 
 def postProcessor(self, measurements):
 self.summary['accessible'] = self.summary.get('accessible', [])
 self.summary['not-accessible'] = self.summary.get('not-accessible', [])
-self.summary['blocked'] = self.summary.get('blocked', [])
+self.summary['blocked'] = self.summary.get('blocked', {})
 
 if self.report['blocking'] not in (False, None):
-self.summary['blocked'].append((self.input,
-self.report['blocking']))
+

[tor-commits] [ooni-probe/master] Remove dead code

[art]

commit 29bbe9bd0b06b674267f0e7af4d1d914cb6fb79d
Author: Arturo Filastò 
Date:   Sun May 8 19:09:39 2016 +0200

Remove dead code
---
 ooni/templates/httpt.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index c58daf2..a4844c8 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -113,7 +113,6 @@ class HTTPTest(NetTestCase):
 config.tor.socks_port))
 
 self.report['socksproxy'] = None
-sockshost, socksport = (None, None)
 if self.localOptions['socksproxy']:
 try:
 sockshost, socksport = 
self.localOptions['socksproxy'].split(':')



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Make the geoip lookup code more robust

[art]

commit 2627cd5ad2058d616cc037cd590b9f64112882a9
Author: Arturo Filastò 
Date:   Mon May 2 17:54:36 2016 +0200

Make the geoip lookup code more robust
---
 ooni/geoip.py | 25 +++--
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/ooni/geoip.py b/ooni/geoip.py
index dcc799e..fa6d1ae 100644
--- a/ooni/geoip.py
+++ b/ooni/geoip.py
@@ -34,25 +34,22 @@ def IPToLocation(ipaddr):
 asn_file = config.get_data_file_path('GeoIP/GeoIPASNum.dat')
 
 location = {'city': None, 'countrycode': 'ZZ', 'asn': 'AS0'}
-
-def error():
+if not asn_file or not country_file:
 log.err("Could not find GeoIP data file in data directories."
 "Try running ooniresources or"
 " edit your ooniprobe.conf")
+return location
 
-try:
-country_dat = GeoIP(country_file)
-location['countrycode'] = country_dat.country_code_by_addr(ipaddr)
-if not location['countrycode']:
-location['countrycode'] = 'ZZ'
-except IOError:
-error()
+country_dat = GeoIP(country_file)
+asn_dat = GeoIP(asn_file)
 
-try:
-asn_dat = GeoIP(asn_file)
-location['asn'] = asn_dat.org_by_addr(ipaddr).split(' ')[0]
-except:
-error()
+country_code = country_dat.country_code_by_addr(ipaddr)
+if country_code is not None:
+location['countrycode'] =  country_code
+
+asn = asn_dat.org_by_addr(ipaddr)
+if asn is not None:
+location['asn'] = asn.split(' ')[0]
 
 return location
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Add support for computing header difference and follow redirects

[art]

commit 8c5fdd48ebb760b72b1b02a83fc96a19f92293c7
Author: Arturo Filastò 
Date:   Sat Apr 16 14:30:29 2016 +0200

Add support for computing header difference and follow redirects
---
 ooni/nettests/blocking/web_connectivity.py | 54 +-
 1 file changed, 39 insertions(+), 15 deletions(-)

diff --git a/ooni/nettests/blocking/web_connectivity.py 
b/ooni/nettests/blocking/web_connectivity.py
index 175c6d3..4b9b6c2 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -81,6 +81,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 }
 requiresRoot = False
 requiresTor = False
+followRedirects = True
 
 # Factor used to determine HTTP blockpage detection
 factor = 0.8
@@ -112,6 +113,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 self.report['client_resolver'] = self.resolverIp
 self.report['dns_consistency'] = None
 self.report['body_length_match'] = None
+self.report['headers_match'] = None
+
 self.report['accessible'] = None
 self.report['blocking'] = None
 
@@ -139,6 +142,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 }
 
 def experiment_dns_query(self):
+log.msg("Doing DNS query for {}".format(self.hostname))
 return self.performALookup(self.hostname)
 
 def tcp_connect(self, socket):
@@ -197,6 +201,22 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 def experiment_http_get_request(self):
 return self.doRequest(self.input, headers=REQUEST_HEADERS)
 
+def compare_headers(self, experiment_http_response):
+count = 0
+control_headers_lower = {k.lower(): v for k, v in
+self.report['control']['http_request']['headers'].items()}
+
+for header_name, header_value in \
+experiment_http_response.headers.getAllRawHeaders():
+try:
+control_headers_lower[header_name.lower()]
+except KeyError:
+log.msg("Did not find the key {}".format(header_name))
+return False
+count += 1
+
+return count == len(self.report['control']['http_request']['headers'])
+
 def compare_body_lengths(self, experiment_http_response):
 control_body_length = self.control['http_request']['body_length']
 experiment_body_length = len(experiment_http_response.body)
@@ -213,10 +233,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 
 self.report['body_proportion'] = rel
 if rel > float(self.factor):
-self.report['body_length_match'] = True
 return True
 else:
-self.report['body_length_match'] = False
 return False
 
 def compare_dns_experiments(self, experiment_dns_answers):
@@ -229,16 +247,13 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 experiment_ips = set(experiment_dns_answers)
 
 if control_ips == experiment_ips:
-self.report['dns_consistency'] = 'consistent'
 return True
 
 for experiment_ip in experiment_ips:
 if is_public_ipv4_address(experiment_ip) is False:
-self.report['dns_consistency'] = 'inconsistent'
 return False
 
 if len(control_ips.intersection(experiment_ips)) > 0:
-self.report['dns_consistency'] = 'consistent'
 return True
 
 experiment_asns = set(map(lambda x: geoip.IPToLocation(x)['asn'],
@@ -247,10 +262,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
control_ips))
 
 if len(control_asns.intersection(experiment_asns)) > 0:
-self.report['dns_consistency'] = 'consistent'
 return True
 
-self.report['dns_consistency'] = 'inconsistent'
 return False
 
 def compare_tcp_experiments(self):
@@ -268,22 +281,26 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 
 def determine_blocking(self, experiment_http_response, 
experiment_dns_answers):
 blocking = False
-body_length_match = None
-dns_consistent = None
-tcp_connect = None
 
 if (self.report['http_experiment_failure'] is None and
 self.report['control']['http_request']['failure'] is None):
-body_length_match = 
self.compare_body_lengths(experiment_http_response)
+self.report['body_length_match'] = self.compare_body_lengths(
+experiment_http_response)
+self.report['headers_match'] = self.compare_headers(
+experiment_http_response)
 
 dns_consistent = self.compare_dns_experiments(experiment_dns_answers)
+if dns_consistent is True:
+self.report['dns_consistency'] = 'consistent'
+else:
+self.report['dns_consistency'] = 'inconsistent'
 

[tor-commits] [ooni-probe/master] Many improvements to web_connectivity test

[art]

commit 7e1ee49e44480f62ac93a9f63aa90a64342df2ed
Author: Arturo Filastò 
Date:   Thu Apr 14 22:54:49 2016 +0200

Many improvements to web_connectivity test

* Run the probe resolver detection inside of the setupClass
* Display a summary at the end of a test run summarising the results
* Make the blocking detection logic more robust
---
 ooni/nettests/blocking/web_connectivity.py | 152 -
 1 file changed, 108 insertions(+), 44 deletions(-)

diff --git a/ooni/nettests/blocking/web_connectivity.py 
b/ooni/nettests/blocking/web_connectivity.py
index d8276b3..320f614 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -8,6 +8,7 @@ from ipaddr import IPv4Address, AddressValueError
 from twisted.internet import reactor
 from twisted.internet.protocol import Factory, Protocol
 from twisted.internet.endpoints import TCP4ClientEndpoint
+from twisted.names import client, dns
 
 from twisted.internet import defer
 from twisted.python import usage
@@ -19,6 +20,17 @@ from ooni.utils.net import StringProducer, BodyReceiver
 from ooni.templates import httpt, dnst
 from ooni.errors import failureToString
 
+REQUEST_HEADERS = {
+'User-Agent': ['Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, '
+   'like Gecko) Chrome/47.0.2526.106 Safari/537.36'],
+'Accept-Language': ['en-US;q=0.8,en;q=0.5'],
+'Accept': ['text/html,application/xhtml+xml,application/xml;q=0.9,'
+   '*/*;q=0.8']
+}
+
+class InvalidControlResponse(Exception):
+pass
+
 class TCPConnectProtocol(Protocol):
 def connectionMade(self):
 self.transport.loseConnection()
@@ -72,6 +84,21 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 
 # Factor used to determine HTTP blockpage detection
 factor = 0.8
+resolverIp = None
+
+@classmethod
+@defer.inlineCallbacks
+def setUpClass(cls):
+try:
+answers = yield client.lookupAddress(
+cls.localOptions['dns-discovery']
+)
+assert len(answers) > 0
+assert len(answers[0]) > 0
+cls.resolverIp = answers[0][0].payload.dottedQuad()
+except Exception as exc:
+log.exception(exc)
+log.err("Failed to lookup the resolver IP address")
 
 def setUp(self):
 """
@@ -82,7 +109,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 if not self.input:
 raise Exception("No input specified")
 
-self.report['client_resolver'] = None
+self.report['client_resolver'] = self.resolverIp
 self.report['dns_consistency'] = None
 self.report['body_length_match'] = None
 self.report['accessible'] = None
@@ -111,9 +138,6 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 }
 }
 
-def dns_discovery(self):
-return self.performALookup(self.localOptions['dns-discovery'])
-
 def experiment_dns_query(self):
 return self.performALookup(self.hostname)
 
@@ -161,11 +185,17 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 finished = defer.Deferred()
 response.deliverBody(BodyReceiver(finished, content_length))
 body = yield finished
-self.control = json.loads(body)
+try:
+self.control = json.loads(body)
+assert 'http_request' in self.control.keys()
+assert 'tcp_connect' in self.control.keys()
+assert 'dns' in self.control.keys()
+except AssertionError, ValueError:
+raise InvalidControlResponse(body)
 self.report['control'] = self.control
 
 def experiment_http_get_request(self):
-return self.doRequest(self.input)
+return self.doRequest(self.input, headers=REQUEST_HEADERS)
 
 def compare_body_lengths(self, experiment_http_response):
 control_body_length = self.control['http_request']['body_length']
@@ -233,27 +263,23 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 return success
 
 def determine_blocking(self, experiment_http_response, 
experiment_dns_answers):
-blocking = None
+blocking = False
 body_length_match = None
 dns_consistent = None
 tcp_connect = None
 
-if self.report['control_failure'] is None and \
-self.report['http_experiment_failure'] is None and \
-self.report['control']['http_request']['failure'] is None:
+if (self.report['http_experiment_failure'] is None and
+self.report['control']['http_request']['failure'] is None):
 body_length_match = 
self.compare_body_lengths(experiment_http_response)
 
-if self.report['control_failure'] is None:
-dns_consistent = 
self.compare_dns_experiments(experiment_dns_answers)
-
-if self.report['control_failure'] is None:
-   

[tor-commits] [ooni-probe/master] Make the blocking detection heuristics more strict

[art]

commit ff77593d59dc7e25e4214e15571d616d7e374dce
Author: Arturo Filastò 
Date:   Mon Apr 18 18:45:09 2016 +0200

Make the blocking detection heuristics more strict
---
 ooni/nettests/blocking/web_connectivity.py | 16 +---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/ooni/nettests/blocking/web_connectivity.py 
b/ooni/nettests/blocking/web_connectivity.py
index 4b9b6c2..28ee942 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -84,7 +84,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 followRedirects = True
 
 # Factor used to determine HTTP blockpage detection
-factor = 0.8
+# the factor 0.7 comes from 
http://www3.cs.stonybrook.edu/~phillipa/papers/JLFG14.pdf
+factor = 0.7
 resolverIp = None
 
 @classmethod
@@ -296,14 +297,23 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 self.report['dns_consistency'] = 'inconsistent'
 tcp_connect = self.compare_tcp_experiments()
 
-if dns_consistent == True and tcp_connect == False:
+if (dns_consistent == True and tcp_connect == False and
+self.report['http_experiment_failure'] is not None):
 blocking = 'tcp_ip'
 
+# XXX we may want to have different codes for these two types of
+# blocking
 elif (dns_consistent == True and tcp_connect == True and
   self.report['body_length_match'] == False):
 blocking = 'http'
+elif (dns_consistent == True and tcp_connect == True and
+self.report['http_experiment_failure'] is not None and
+self.report['control']['http_request']['failure'] != 
self.report['http_experiment_failure']):
+blocking = 'http'
 
-elif dns_consistent == False:
+elif (dns_consistent == False and
+  (self.report['http_experiment_failure'] is not None or
+   self.report['body_length_match'] == False)):
 blocking = 'dns'
 
 return blocking



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] [feature] Add setUpClass method to NetTestCase

[art]

commit 9367f7f0398cfd0e2105596719c47009a5e65924
Author: Arturo Filastò 
Date:   Thu Apr 14 22:54:25 2016 +0200

[feature] Add setUpClass method to NetTestCase
---
 ooni/director.py   |  2 +-
 ooni/nettest.py| 18 +-
 ooni/tests/test_nettest.py |  6 --
 3 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/ooni/director.py b/ooni/director.py
index 64b5a98..43ca36c 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -257,7 +257,7 @@ class Director(object):
 net_test = NetTest(test_cases, test_details, report)
 net_test.director = self
 
-yield net_test.initializeInputProcessor()
+yield net_test.initialize()
 try:
 self.activeNetTests.append(net_test)
 self.measurementManager.schedule(net_test.generateMeasurements())
diff --git a/ooni/nettest.py b/ooni/nettest.py
index d01b3e4..67d3e27 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -566,14 +566,20 @@ class NetTest(object):
 return measurement
 
 @defer.inlineCallbacks
-def initializeInputProcessor(self):
+def initialize(self):
 for test_class, _ in self.testCases:
+# Initialize Input Processor
 test_class.inputs = yield defer.maybeDeferred(
 test_class().getInputProcessor
 )
 if not test_class.inputs:
 test_class.inputs = [None]
 
+# Run the setupClass method
+yield defer.maybeDeferred(
+test_class.setUpClass
+)
+
 def generateMeasurements(self):
 """
 This is a generator that yields measurements and registers the
@@ -701,6 +707,16 @@ class NetTestCase(object):
 
 localOptions = {}
 
+@classmethod
+def setUpClass(cls):
+"""
+You can override this hook with logic that should be run once before
+any test method in the NetTestCase is run.
+This can be useful to populate class attribute that should be valid
+for all the runtime of the NetTest.
+"""
+pass
+
 def _setUp(self):
 """
 This is the internal setup method to be overwritten by templates.
diff --git a/ooni/tests/test_nettest.py b/ooni/tests/test_nettest.py
index 91e663f..660fd3e 100644
--- a/ooni/tests/test_nettest.py
+++ b/ooni/tests/test_nettest.py
@@ -253,12 +253,13 @@ class TestNetTest(unittest.TestCase):
 ntl.loadNetTestString(net_test_string_with_required_option)
 self.assertRaises(MissingRequiredOption, ntl.checkOptions)
 
+@defer.inlineCallbacks
 def test_net_test_inputs(self):
 ntl = NetTestLoader(dummyArgsWithFile)
 ntl.loadNetTestString(net_test_string_with_file)
 ntl.checkOptions()
 nt = NetTest(ntl.getTestCases(), ntl.getTestDetails(), None)
-nt.initializeInputProcessor()
+yield nt.initialize()
 
 # XXX: if you use the same test_class twice you will have consumed all
 # of its inputs!
@@ -275,6 +276,7 @@ class TestNetTest(unittest.TestCase):
 ntl.checkOptions()
 self.assertEqual(dict(ntl.localOptions), dummyOptions)
 
+@defer.inlineCallbacks
 def test_generate_measurements_size(self):
 ntl = NetTestLoader(dummyArgsWithFile)
 ntl.loadNetTestString(net_test_string_with_file)
@@ -282,7 +284,7 @@ class TestNetTest(unittest.TestCase):
 
 net_test = NetTest(ntl.getTestCases(), ntl.getTestDetails(), None)
 
-net_test.initializeInputProcessor()
+yield net_test.initialize()
 measurements = list(net_test.generateMeasurements())
 self.assertEqual(len(measurements), 20)
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Implement unittest to check if tests that fail twice

[art]

commit cf9ea6cb6f1cbd5455651291c8fe2c6822c804eb
Author: Arturo Filastò 
Date:   Tue May 3 12:27:54 2016 +0200

Implement unittest to check if tests that fail twice

This is to ensure that net tests that eventually succeed, but fail sometimes
will run properly until the end.
---
 ooni/tests/test_director.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ooni/tests/test_director.py b/ooni/tests/test_director.py
index 5875ccb..d2c4c82 100644
--- a/ooni/tests/test_director.py
+++ b/ooni/tests/test_director.py
@@ -5,6 +5,8 @@ from ooni.director import Director
 from ooni.nettest import NetTestLoader
 from ooni.tests.bases import ConfigTestCase
 
+from ooni.nettest import NetTestLoader
+
 from twisted.internet import defer
 from twisted.trial import unittest
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Also record all the redirects

[art]

commit 8a745093ecd27240053b7e251de46eb4c303df6f
Author: Arturo Filastò 
Date:   Mon Apr 18 18:45:29 2016 +0200

Also record all the redirects

* Add monkey patch for bug in twisted RedirectAgent: 
https://twistedmatrix.com/trac/ticket/8265
---
 ooni/templates/httpt.py   | 22 ++
 ooni/utils/trueheaders.py | 42 +-
 2 files changed, 55 insertions(+), 9 deletions(-)

diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index 2b280f0..2a17f5b 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -7,6 +7,7 @@ from twisted.web.client import readBody, PartialDownloadError
 from twisted.internet import reactor
 from twisted.internet.endpoints import TCP4ClientEndpoint
 from ooni.utils.trueheaders import TrueHeadersAgent, TrueHeadersSOCKS5Agent
+from ooni.utils.trueheaders import FixedRedirectAgent
 
 from ooni.nettest import NetTestCase
 from ooni.utils import log, base64Dict
@@ -37,8 +38,15 @@ class StreamListener(StreamListenerMixin):
 log.err("Tor Exit ip detection failed")
 
 
+def _representHeaders(headers):
+represented_headers = {}
+for name, value in headers.getAllRawHeaders():
+represented_headers[name] = value[0]
+return represented_headers
 
 def _representBody(body):
+if not body:
+return body
 # XXX perhaps add support for decoding gzip in the future.
 body = body.replace('\0', '')
 decoded = False
@@ -115,9 +123,8 @@ class HTTPTest(NetTestCase):
 
 if self.followRedirects:
 try:
-from twisted.web.client import RedirectAgent
-self.control_agent = RedirectAgent(self.control_agent)
-self.agent = RedirectAgent(self.agent)
+self.control_agent = FixedRedirectAgent(self.control_agent)
+self.agent = FixedRedirectAgent(self.agent)
 self.report['agent'] = 'redirect'
 except:
 log.err("Warning! You are running an old version of twisted"\
@@ -147,11 +154,10 @@ class HTTPTest(NetTestCase):
 
 failure (instance): An instance of 
:class:twisted.internet.failure.Failure
 """
-def _representHeaders(headers):
-represented_headers = {}
-for name, value in headers.getAllRawHeaders():
-represented_headers[name] = value[0]
-return represented_headers
+if response and response.previousResponse:
+self.addToReport(request, response.previousResponse,
+ response_body=None,
+ failure_string=None)
 
 log.debug("Adding %s to report" % request)
 request_headers = TrueHeaders(request['headers'])
diff --git a/ooni/utils/trueheaders.py b/ooni/utils/trueheaders.py
index d5aa206..a5b2fe3 100644
--- a/ooni/utils/trueheaders.py
+++ b/ooni/utils/trueheaders.py
@@ -7,9 +7,14 @@
 import itertools
 from copy import copy
 
-from twisted.web import client, _newclient, http_headers
+from twisted.python.failure import Failure
+
+from twisted.web import client, _newclient, http_headers, error
+
 from twisted.web._newclient import RequestNotSent, RequestGenerationFailed
 from twisted.web._newclient import TransportProxyProducer, STATUS
+from twisted.web._newclient import ResponseFailed
+
 from twisted.internet import reactor
 from twisted.internet.defer import Deferred, fail, maybeDeferred, failure
 
@@ -179,3 +184,38 @@ class TrueHeadersSOCKS5Agent(SOCKS5Agent):
 self._wrappedAgent._pool = pool
 else:
 self._pool = pool
+
+class FixedRedirectAgent(client.BrowserLikeRedirectAgent):
+"""
+This is a redirect agent with this patch manually applied:
+https://twistedmatrix.com/trac/ticket/8265
+"""
+def _handleRedirect(self, response, method, uri, headers, redirectCount):
+"""
+Handle a redirect response, checking the number of redirects already
+followed, and extracting the location header fields.
+
+This is pathed to fix a bug in infinite redirect loop.
+"""
+if redirectCount >= self._redirectLimit:
+err = error.InfiniteRedirection(
+response.code,
+b'Infinite redirection detected',
+location=uri)
+raise ResponseFailed([Failure(err)], response)
+locationHeaders = response.headers.getRawHeaders(b'location', [])
+if not locationHeaders:
+err = error.RedirectWithNoLocation(
+response.code, b'No location header field', uri)
+raise ResponseFailed([Failure(err)], response)
+location = self._resolveLocation(response.request.absoluteURI, 
locationHeaders[0])
+deferred = self._agent.request(method, location, headers)
+
+def _chainResponse(newResponse):
+newResponse.setPreviousResponse(response)
+return 

[tor-commits] [ooni-probe/master] Progress on implementing web_connectivity test

[art]

commit 9ab068b324290df91d4296f2f3388071df3bbac4
Author: Arturo Filastò 
Date:   Tue Feb 2 16:07:04 2016 +0100

Progress on implementing web_connectivity test
---
 ooni/nettests/blocking/web_connectivity.py | 62 --
 1 file changed, 42 insertions(+), 20 deletions(-)

diff --git a/ooni/nettests/blocking/web_connectivity.py 
b/ooni/nettests/blocking/web_connectivity.py
index bcc3a28..bc23f6d 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -13,7 +13,7 @@ from twisted.python import usage
 from ooni.utils import log
 
 from ooni.utils.net import StringProducer, BodyReceiver
-from ooni.templates import httpt, dnst, tcpt
+from ooni.templates import httpt, dnst
 from ooni.errors import failureToString
 
 class TCPConnectProtocol(Protocol):
@@ -21,6 +21,7 @@ class TCPConnectProtocol(Protocol):
 self.transport.loseConnection()
 
 class TCPConnectFactory(Factory):
+noisy = False
 def buildProtocol(self, addr):
 return TCPConnectProtocol()
 
@@ -74,6 +75,9 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 self.report['accessible'] = None
 self.report['blocking'] = None
 
+self.report['control_failure'] = None
+self.report['experiment_failure'] = None
+
 self.report['tcp_connect'] = [
 ]
 
@@ -83,8 +87,10 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 
 self.control = {
 'tcp_connect': {},
-'dns_consistency': [],
-'http_requests': {
+'dns': {
+'ips': []
+},
+'http_request': {
 'body_length': None,
 'headers': {}
 }
@@ -93,7 +99,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 def dns_discovery(self):
 return self.performALookup(self.localOptions['dns-discovery'])
 
-def dns_consistency(self):
+def experiment_dns_query(self):
 return self.performALookup(self.hostname)
 
 def tcp_connect(self, socket):
@@ -119,6 +125,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 @d.addErrback
 def eb(failure):
 result['status']['success'] = False
+result['status']['failure'] = failureToString(failure)
 self.report['tcp_connect'].append(result)
 return d
 
@@ -144,11 +151,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 def experiment_http_get_request(self):
 return self.doRequest(self.input)
 
-def compare_control_experiment(self, experiment_http_response,
-   experiment_dns_answers):
-
-blocking = None
-
+def compare_body_lengths(self, experiment_http_response):
 control_body_length = self.control['http_request']['body_length']
 experiment_body_length = len(experiment_http_response.body)
 
@@ -165,20 +168,22 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 self.report['body_proportion'] = rel
 if rel > float(self.factor):
 self.report['body_length_match'] = True
+return None
 else:
-blocking = 'http'
 self.report['body_length_match'] = False
+return 'http'
 
+def compare_dns_experiments(self, experiment_dns_answers):
 control_ips = set(self.control['dns']['ips'])
 experiment_ips = set(experiment_dns_answers)
 
 if len(control_ips.intersection(experiment_ips)) > 0:
 self.report['dns_consistency'] = 'consistent'
 else:
-if blocking is not None:
-blocking = 'dns'
 self.report['dns_consistency'] = 'inconsistent'
 
+def compare_tcp_experiments(self):
+blocking = False
 for idx, result in enumerate(self.report['tcp_connect']):
 socket = "%s:%s" % (result['ip'], result['port'])
 control_status = self.control['tcp_connect'][socket]
@@ -189,14 +194,13 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 blocking = 'tcp_ip'
 else:
 self.report['tcp_connect'][idx]['status']['blocked'] = False
-
-self.report['blocking'] = blocking
+return blocking
 
 @defer.inlineCallbacks
 def test_web_connectivity(self):
 results = yield defer.DeferredList([
 self.dns_discovery(),
-self.dns_consistency()
+self.experiment_dns_query()
 ])
 
 self.report['client_resolver'] = None
@@ -207,11 +211,29 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 
 sockets = map(lambda x: "%s:80" % x, results[1][1])
 
-dl = [self.control_request(sockets)]
+control_request = self.control_request(sockets)
+@control_request.addErrback
+def control_err(failure):
+self.report['control_failure'] = 

[tor-commits] [ooni-probe/master] Add check on equality of control and experiment DNS responses

[art]

commit 6a4ed0161a961daaaccfd8b082a7328683f3b2fc
Author: Arturo Filastò 
Date:   Fri Apr 15 00:05:07 2016 +0200

Add check on equality of control and experiment DNS responses
---
 ooni/nettests/blocking/web_connectivity.py | 4 
 1 file changed, 4 insertions(+)

diff --git a/ooni/nettests/blocking/web_connectivity.py 
b/ooni/nettests/blocking/web_connectivity.py
index 320f614..175c6d3 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -228,6 +228,10 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 control_ips = set(self.control['dns']['ips'])
 experiment_ips = set(experiment_dns_answers)
 
+if control_ips == experiment_ips:
+self.report['dns_consistency'] = 'consistent'
+return True
+
 for experiment_ip in experiment_ips:
 if is_public_ipv4_address(experiment_ip) is False:
 self.report['dns_consistency'] = 'inconsistent'



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Maintain backward compatibility

[art]

commit d9656bc7187a10774bac3851b33fe4118b10c7a9
Author: Arturo Filastò 
Date:   Sat Jan 30 12:36:49 2016 +0100

Maintain backward compatibility
---
 ooni/reporter.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ooni/reporter.py b/ooni/reporter.py
index 63d6f15..66a3e86 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -199,6 +199,7 @@ class YAMLReporter(OReporter):
 raise Exception("Failed to serialise entry")
 content += safe_dump(report_entry)
 content += '...\n'
+report_entry.update(self.testDetails)
 self._write(content)
 
 def createReport(self):
@@ -292,6 +293,7 @@ class OONIBReporter(OReporter):
 report_entry = entry
 else:
 raise Exception("Failed to serialise entry")
+report_entry.update(self.testDetails)
 content += safe_dump(report_entry)
 content += '...\n'
 return content



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Improve charset detection regexp

[art]

commit 55cd85351c02d7a7333f1c39e9993820b05b2b6d
Author: Arturo Filastò 
Date:   Sun May 8 19:08:22 2016 +0200

Improve charset detection regexp
---
 ooni/templates/httpt.py  | 12 ++--
 ooni/tests/test_templates.py |  3 +++
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index 51fba1a..6b4c4b9 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -1,4 +1,5 @@
 import re
+import codecs
 import random
 
 from txtorcon.interface import StreamListenerMixin
@@ -17,7 +18,7 @@ from ooni.utils.net import StringProducer, userAgents
 from ooni.utils.trueheaders import TrueHeaders
 from ooni.errors import handleAllFailures
 
-META_CHARSET_REGEXP = 
re.compile(']*?charset\s*=[\s"\']*([^\s"\'/>]+)')
+META_CHARSET_REGEXP = 
re.compile(']*?charset\s*=[\s"\']*([^\s"\'/>!;]+)')
 
 class InvalidSocksProxyOption(Exception):
 pass
@@ -56,7 +57,14 @@ def _representBody(body):
 # try to decode using that one first
 charset = META_CHARSET_REGEXP.search(body, re.IGNORECASE)
 if charset:
-charsets.insert(0, charset.group(1))
+try:
+encoding = charset.group(1).lower()
+codecs.lookup(encoding)
+charsets.insert(0, encoding)
+except (LookupError, IndexError):
+# Skip invalid codecs and partial regexp match
+pass
+
 for encoding in charsets:
 try:
 body = unicode(body, encoding)
diff --git a/ooni/tests/test_templates.py b/ooni/tests/test_templates.py
index e8fe636..ebd5b2e 100644
--- a/ooni/tests/test_templates.py
+++ b/ooni/tests/test_templates.py
@@ -55,8 +55,11 @@ class TestHTTPT(unittest.TestCase):
 """
 with_charset_html = no_charset_html + '\n'
 with_empty_charset = no_charset_html + '\n'
+with_two_charsets = no_charset_html + '\n'
 self.assertEqual(httpt.META_CHARSET_REGEXP.search(no_charset_html), 
None)
 
self.assertEqual(httpt.META_CHARSET_REGEXP.search(with_charset_html).group(1), 
'iso-8859-1')
+self.assertEqual(httpt.META_CHARSET_REGEXP.search(
+with_two_charsets).group(1), 'UTF-8')
 self.assertEqual(httpt.META_CHARSET_REGEXP.search(with_empty_charset), 
None)
 
 class TestDNST(unittest.TestCase):



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] First stab at implementing the web_connectivity test

[art]

commit a710467d95627c196f78f553e11055e16d9b9e7f
Author: Arturo Filastò 
Date:   Mon Feb 1 17:29:08 2016 +0100

First stab at implementing the web_connectivity test
---
 ooni/nettests/blocking/web_connectivity.py | 217 +
 1 file changed, 217 insertions(+)

diff --git a/ooni/nettests/blocking/web_connectivity.py 
b/ooni/nettests/blocking/web_connectivity.py
new file mode 100644
index 000..bcc3a28
--- /dev/null
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -0,0 +1,217 @@
+# -*- encoding: utf-8 -*-
+
+import json
+from urlparse import urlparse
+
+from twisted.internet import reactor
+from twisted.internet.protocol import Factory, Protocol
+from twisted.internet.endpoints import TCP4ClientEndpoint
+
+from twisted.internet import defer
+from twisted.python import usage
+
+from ooni.utils import log
+
+from ooni.utils.net import StringProducer, BodyReceiver
+from ooni.templates import httpt, dnst, tcpt
+from ooni.errors import failureToString
+
+class TCPConnectProtocol(Protocol):
+def connectionMade(self):
+self.transport.loseConnection()
+
+class TCPConnectFactory(Factory):
+def buildProtocol(self, addr):
+return TCPConnectProtocol()
+
+
+class UsageOptions(usage.Options):
+optParameters = [
+['url', 'u', None, 'Specify a single URL to test'],
+['dns-discovery', 'd', None, 'Specify the dns discovery test helper'],
+['backend', 'b', None, 'The web_consistency backend test helper'],
+]
+
+
+class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
+"""
+Web connectivity
+"""
+name = "Web connectivity"
+description = ("Performs a HTTP GET request over Tor and one over the "
+  "local network and compares the two results.")
+author = "Arturo Filastò"
+version = "0.1.0"
+
+usageOptions = UsageOptions
+
+inputFile = [
+'file', 'f', None, 'List of URLS to perform GET requests to'
+]
+
+requiredTestHelpers = {
+'backend': 'web_connectivity',
+'dns-discovery': 'dns_discovery'
+}
+requiresRoot = False
+requiresTor = False
+
+# Factor used to determine HTTP blockpage detection
+factor = 0.8
+
+def setUp(self):
+"""
+Check for inputs.
+"""
+if self.localOptions['url']:
+self.input = self.localOptions['url']
+if not self.input:
+raise Exception("No input specified")
+
+self.report['client_resolver'] = None
+self.report['dns_consistency'] = None
+self.report['body_length_match'] = None
+self.report['accessible'] = None
+self.report['blocking'] = None
+
+self.report['tcp_connect'] = [
+]
+
+self.hostname = urlparse(self.input).netloc
+if not self.hostname:
+raise Exception("Invalid input")
+
+self.control = {
+'tcp_connect': {},
+'dns_consistency': [],
+'http_requests': {
+'body_length': None,
+'headers': {}
+}
+}
+
+def dns_discovery(self):
+return self.performALookup(self.localOptions['dns-discovery'])
+
+def dns_consistency(self):
+return self.performALookup(self.hostname)
+
+def tcp_connect(self, socket):
+ip_address, port = socket.split(":")
+port = int(port)
+result = {
+'ip': ip_address,
+'port': port,
+'status': {
+'success': None,
+'failure': None,
+'blocked': None
+}
+}
+point = TCP4ClientEndpoint(reactor, ip_address, port)
+d = point.connect(TCPConnectFactory())
+@d.addCallback
+def cb(p):
+result['status']['success'] = True
+result['status']['blocked'] = False
+self.report['tcp_connect'].append(result)
+
+@d.addErrback
+def eb(failure):
+result['status']['success'] = False
+self.report['tcp_connect'].append(result)
+return d
+
+@defer.inlineCallbacks
+def control_request(self, sockets):
+bodyProducer = StringProducer(json.dumps({
+'http_request': self.input,
+'tcp_connect': sockets
+}))
+response = yield self.agent.request("POST",
+str(self.localOptions['backend']),
+bodyProducer=bodyProducer)
+try:
+content_length = 
int(response.headers.getRawHeaders('content-length')[0])
+except Exception:
+content_length = None
+
+finished = defer.Deferred()
+response.deliverBody(BodyReceiver(finished, content_length))
+body = yield finished
+self.control = json.loads(body)
+
+def experiment_http_get_request(self):
+return self.doRequest(self.input)
+
+def 

[tor-commits] [ooni-probe/master] Update the tests to be consistent with the new data formats

[art]

commit ff21910bcb2d2a3e54e5fc33be6b6d73ed0b60ee
Author: Arturo Filastò 
Date:   Thu Jan 28 13:09:36 2016 +0100

Update the tests to be consistent with the new data formats
---
 ooni/templates/httpt.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index e608f8f..73bd5db 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -183,6 +183,7 @@ class HTTPTest(NetTestCase):
 session['failure'] = None
 if failure_string:
 session['failure'] = failure_string
+
 self.report['requests'].append(session)
 
 def _processResponseBody(self, response_body, request, response, 
body_processor):



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] [refactoring] Change signature of Deck to be more robust

[art]

commit aa8314973df0f6e458aa1e0de786faaad8264dc3
Author: Arturo Filastò 
Date:   Fri Apr 15 15:45:03 2016 +0200

[refactoring] Change signature of Deck to be more robust

Do more sanity checks on bouncer address at instantiation time
---
 ooni/deck.py | 30 ++
 ooni/oonibclient.py  | 15 +--
 ooni/oonicli.py  |  4 ++--
 ooni/tests/mocks.py  |  3 +++
 ooni/tests/test_deck.py  | 14 +++---
 ooni/tests/test_templates.py |  3 +++
 6 files changed, 42 insertions(+), 27 deletions(-)

diff --git a/ooni/deck.py b/ooni/deck.py
index b13385d..4b2c502 100644
--- a/ooni/deck.py
+++ b/ooni/deck.py
@@ -95,28 +95,27 @@ def nettest_to_path(path, allow_arbitrary_paths=False):
 
 
 class Deck(InputFile):
+# this exists so we can mock it out in unittests
+_OONIBClient = OONIBClient
+
 def __init__(self, deck_hash=None,
- deckFile=None,
+ bouncer=None,
  decks_directory=config.decks_directory,
  no_collector=False):
 self.id = deck_hash
-self.requiresTor = False
 self.no_collector = no_collector
-self.bouncer = ''
+self.bouncer = bouncer
+
+self.requiresTor = False
+
 self.netTestLoaders = []
 self.inputs = []
 
-self.oonibclient = OONIBClient(self.bouncer)
-
 self.decksDirectory = os.path.abspath(decks_directory)
-self.deckHash = deck_hash
-
-if deckFile:
-self.loadDeck(deckFile)
 
 @property
 def cached_file(self):
-return os.path.join(self.decksDirectory, self.deckHash)
+return os.path.join(self.decksDirectory, self.id)
 
 @property
 def cached_descriptor(self):
@@ -124,7 +123,7 @@ class Deck(InputFile):
 
 def loadDeck(self, deckFile):
 with open(deckFile) as f:
-self.deckHash = sha256(f.read()).hexdigest()
+self.id = sha256(f.read()).hexdigest()
 f.seek(0)
 test_deck = yaml.safe_load(f)
 
@@ -175,7 +174,7 @@ class Deck(InputFile):
 
 @defer.inlineCallbacks
 def lookupCollectorAndTestHelpers(self):
-self.oonibclient.address = self.bouncer
+oonibclient = self._OONIBClient(self.bouncer)
 
 required_nettests = []
 
@@ -201,8 +200,7 @@ class Deck(InputFile):
 if not requires_test_helpers and not requires_collector:
 defer.returnValue(None)
 
-log.debug("Looking up {}".format(required_nettests))
-response = yield 
self.oonibclient.lookupTestCollector(required_nettests)
+response = yield oonibclient.lookupTestCollector(required_nettests)
 provided_net_tests = response['net-tests']
 
 def find_collector_and_test_helpers(test_name, test_version, 
input_files):
@@ -240,10 +238,10 @@ class Deck(InputFile):
 for i in net_test_loader.inputFiles:
 if i['url']:
 log.debug("Downloading %s" % i['url'])
-self.oonibclient.address = i['address']
+oonibclient = self._OONIBClient(i['address'])
 
 try:
-input_file = yield 
self.oonibclient.downloadInput(i['hash'])
+input_file = yield oonibclient.downloadInput(i['hash'])
 except:
 raise e.UnableToLoadDeckInput
 
diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py
index 388bb68..336ae4e 100644
--- a/ooni/oonibclient.py
+++ b/ooni/oonibclient.py
@@ -1,5 +1,7 @@
 import json
 
+from urlparse import urljoin
+
 from twisted.web.client import Agent
 from twisted.internet import defer, reactor
 from twisted.internet.endpoints import TCP4ClientEndpoint
@@ -15,6 +17,15 @@ class OONIBClient(object):
 retries = 3
 
 def __init__(self, address):
+if address.startswith("https://;):
+log.err("HTTPS bouncers are currently not supported!")
+raise e.InvalidOONIBBouncerAddress
+elif address.startswith("http://;):
+log.msg("Warning using plaintext bouncer!")
+elif address.startswith("httpo://"):
+log.debug("Using Tor hidden service bouncer: {}".format(address))
+else:
+raise e.InvalidOONIBBouncerAddress
 self.address = address
 
 def _request(self, method, urn, genReceiver, bodyProducer=None):
@@ -30,7 +41,7 @@ class OONIBClient(object):
 raise e.InvalidOONIBBouncerAddress
 
 elif self.address.startswith('http://'):
-log.msg("Warning using unencrypted backend")
+log.msg("Warning using unencrypted bouncer")
 agent = Agent(reactor)
 
 attempts = 0
@@ -38,7 +49,7 @@ class OONIBClient(object):
 finished = defer.Deferred()
 
 def perform_request(attempts):
-uri = address + urn
+uri = urljoin(address, urn)
 d = agent.request(method, uri, 

[tor-commits] [ooni-probe/master] Add detection for accessibility of the website

[art]

commit bcd87425f035a2f6d37278007a668cd5a0349bf8
Author: Arturo Filastò 
Date:   Thu Feb 4 16:26:16 2016 +0100

Add detection for accessibility of the website

* Minor refactoring of the test
---
 ooni/nettests/blocking/web_connectivity.py | 91 --
 1 file changed, 60 insertions(+), 31 deletions(-)

diff --git a/ooni/nettests/blocking/web_connectivity.py 
b/ooni/nettests/blocking/web_connectivity.py
index 8d83ad9..a61819a 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -89,7 +89,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 self.report['blocking'] = None
 
 self.report['control_failure'] = None
-self.report['experiment_failure'] = None
+self.report['http_experiment_failure'] = None
+self.report['dns_experiment_failure'] = None
 
 self.report['tcp_connect'] = []
 self.report['control'] = {}
@@ -188,6 +189,11 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 return False
 
 def compare_dns_experiments(self, experiment_dns_answers):
+if self.control['dns']['failure'] is not None and \
+self.control['dns']['failure'] == 
self.report['dns_experiment_failure']:
+self.report['dns_consistency'] = 'consistent'
+return True
+
 control_ips = set(self.control['dns']['ips'])
 experiment_ips = set(experiment_dns_answers)
 
@@ -225,11 +231,47 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 self.report['tcp_connect'][idx]['status']['blocked'] = False
 return success
 
+def determine_blocking(self, experiment_http_response, 
experiment_dns_answers):
+blocking = None
+body_length_match = None
+dns_consistent = None
+tcp_connect = None
+
+if self.report['control_failure'] is None and \
+self.report['http_experiment_failure'] is None:
+body_length_match = 
self.compare_body_lengths(experiment_http_response)
+
+if self.report['control_failure'] is None:
+dns_consistent = 
self.compare_dns_experiments(experiment_dns_answers)
+
+if self.report['control_failure'] is None:
+tcp_connect = self.compare_tcp_experiments()
+
+if dns_consistent == True and tcp_connect == False:
+blocking = 'tcp_ip'
+
+elif dns_consistent == True and \
+tcp_connect == True and body_length_match == False:
+blocking = 'http'
+
+elif dns_consistent == False:
+blocking = 'dns'
+
+return blocking
+
+
 @defer.inlineCallbacks
 def test_web_connectivity(self):
+experiment_dns = self.experiment_dns_query()
+
+@experiment_dns.addErrback
+def dns_experiment_err(failure):
+self.report['dns_experiment_failure'] = failureToString(failure)
+return []
+
 results = yield defer.DeferredList([
 self.dns_discovery(),
-self.experiment_dns_query()
+experiment_dns
 ])
 
 self.report['client_resolver'] = None
@@ -254,38 +296,25 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 
 experiment_http = self.experiment_http_get_request()
 @experiment_http.addErrback
-def experiment_err(failure):
-self.report['experiment_failure'] = failureToString(failure)
+def http_experiment_err(failure):
+self.report['http_experiment_failure'] = failureToString(failure)
 
 experiment_http_response = yield experiment_http
 
-blocking = None
-body_length_match = None
-dns_consistent = None
-tcp_connect = None
-
-if self.report['control_failure'] is None and \
-self.report['experiment_failure'] is None:
-body_length_match = 
self.compare_body_lengths(experiment_http_response)
-
-if self.report['control_failure'] is None:
-dns_consistent = 
self.compare_dns_experiments(experiment_dns_answers)
-
-if self.report['control_failure'] is None:
-tcp_connect = self.compare_tcp_experiments()
-
-if dns_consistent == True and tcp_connect == False:
-blocking = 'tcp_ip'
-
-elif dns_consistent == True and \
-tcp_connect == True and body_length_match == False:
-blocking = 'http'
-
-elif dns_consistent == False:
-blocking = 'dns'
-
+blocking = self.determine_blocking(experiment_http_response, 
experiment_dns_answers)
 self.report['blocking'] = blocking
+
 if blocking is not None:
-log.msg("Blocking detected on %s due to %s" % (self.input, 
blocking))
+log.msg("%s: BLOCKING DETECTED due to %s" % (self.input, blocking))
+else:
+log.msg("%s: No blocking detected" % self.input)
+

[tor-commits] [ooni-probe/master] Clean up how we handle the reportId and remove unused functions

[art]

commit da51f6d069ac55ebf430ebb0adfa6a70110fda68
Author: Arturo Filastò 
Date:   Tue Apr 12 17:09:51 2016 +0200

Clean up how we handle the reportId and remove unused functions
---
 ooni/deck.py| 52 ---
 ooni/director.py|  8 +-
 ooni/reporter.py| 66 +++--
 ooni/tasks.py   |  3 ---
 ooni/tests/mocks.py | 19 +
 ooni/tests/test_deck.py |  4 +--
 ooni/tests/test_reporter.py |  2 +-
 7 files changed, 55 insertions(+), 99 deletions(-)

diff --git a/ooni/deck.py b/ooni/deck.py
index 4b6d894..b13385d 100644
--- a/ooni/deck.py
+++ b/ooni/deck.py
@@ -171,10 +171,10 @@ class Deck(InputFile):
 
 if self.bouncer:
 log.msg("Looking up collector and test helpers")
-yield self.lookupCollector()
+yield self.lookupCollectorAndTestHelpers()
 
 @defer.inlineCallbacks
-def lookupCollector(self):
+def lookupCollectorAndTestHelpers(self):
 self.oonibclient.address = self.bouncer
 
 required_nettests = []
@@ -221,9 +221,9 @@ class Deck(InputFile):
 net_test_loader.testName)
 
 collector, test_helpers = \
-find_collector_and_test_helpers(net_test_loader.testName,
-net_test_loader.testVersion,
-net_test_loader.inputFiles)
+
find_collector_and_test_helpers(test_name=net_test_loader.testName,
+
test_version=net_test_loader.testVersion,
+
input_files=net_test_loader.inputFiles)
 
 for option, name in net_test_loader.missingTestHelpers:
 test_helper_address = test_helpers[name].encode('utf-8')
@@ -234,48 +234,6 @@ class Deck(InputFile):
 net_test_loader.collector = collector.encode('utf-8')
 
 @defer.inlineCallbacks
-def lookupTestHelpers(self):
-self.oonibclient.address = self.bouncer
-
-required_test_helpers = []
-requires_collector = []
-for net_test_loader in self.netTestLoaders:
-if not net_test_loader.collector and not self.no_collector:
-requires_collector.append(net_test_loader)
-
-required_test_helpers += map(lambda x: x[1],
-   net_test_loader.missingTestHelpers)
-
-if not required_test_helpers and not requires_collector:
-defer.returnValue(None)
-
-response = yield 
self.oonibclient.lookupTestHelpers(required_test_helpers)
-
-for net_test_loader in self.netTestLoaders:
-log.msg("Setting collector and test helpers for %s" %
-net_test_loader.testName)
-
-# Only set the collector if the no collector has been specified
-# from the command line or via the test deck.
-if len(net_test_loader.missingTestHelpers) == 0 and \
-net_test_loader in requires_collector:
-log.msg("Using the default collector: %s" %
-response['default']['collector'])
-net_test_loader.collector = 
response['default']['collector'].encode('utf-8')
-continue
-
-for option, name in net_test_loader.missingTestHelpers:
-test_helper_address = response[name]['address'].encode('utf-8')
-test_helper_collector = \
-response[name]['collector'].encode('utf-8')
-
-log.msg("Using this helper: %s" % test_helper_address)
-net_test_loader.localOptions[option] = test_helper_address
-net_test_loader.testHelpers[option] = test_helper_address
-if net_test_loader in requires_collector:
-net_test_loader.collector = test_helper_collector
-
-@defer.inlineCallbacks
 def fetchAndVerifyNetTestInput(self, net_test_loader):
 """ fetch and verify a single NetTest's inputs """
 log.debug("Fetching and verifying inputs")
diff --git a/ooni/director.py b/ooni/director.py
index 13a4503..64b5a98 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -253,16 +253,10 @@ class Director(object):
 self.reportEntryManager, collector_address,
 no_yamloo)
 
+yield report.open()
 net_test = NetTest(test_cases, test_details, report)
 net_test.director = self
 
-yield net_test.report.open()
-
-# XXX this needs some serious refactoring
-net_test_loader.reportID = report.reportID
-net_test.reportID = report.reportID
-net_test.testDetails['report_id'] = report.reportID
-
 yield net_test.initializeInputProcessor()
 try:
 

[tor-commits] [ooni-probe/master] Implement basic web connectivity test

[art]

commit d92a95b628167043a84ceabac77e45f3cd319d3d
Author: Arturo Filastò 
Date:   Wed Feb 3 19:36:11 2016 +0100

Implement basic web connectivity test

* Various hacky fixes to enforce correct report format and handling of 
report_id
---
 ooni/director.py   |  5 ++
 ooni/nettest.py|  6 ++-
 ooni/nettests/blocking/web_connectivity.py | 86 --
 ooni/reporter.py   |  7 ++-
 ooni/tasks.py  |  3 ++
 ooni/templates/httpt.py| 19 +++
 6 files changed, 94 insertions(+), 32 deletions(-)

diff --git a/ooni/director.py b/ooni/director.py
index 02619c2..13a4503 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -258,6 +258,11 @@ class Director(object):
 
 yield net_test.report.open()
 
+# XXX this needs some serious refactoring
+net_test_loader.reportID = report.reportID
+net_test.reportID = report.reportID
+net_test.testDetails['report_id'] = report.reportID
+
 yield net_test.initializeInputProcessor()
 try:
 self.activeNetTests.append(net_test)
diff --git a/ooni/nettest.py b/ooni/nettest.py
index 074e8c2..166451a 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -161,6 +161,8 @@ class NetTestLoader(object):
 method_prefix = 'test'
 collector = None
 yamloo = True
+requiresTor = False
+reportID = None
 
 def __init__(self, options, test_file=None, test_string=None,
  annotations={}):
@@ -568,10 +570,12 @@ class NetTest(object):
 
 @defer.inlineCallbacks
 def initializeInputProcessor(self):
-for test_class, test_method in self.testCases:
+for test_class, _ in self.testCases:
 test_class.inputs = yield defer.maybeDeferred(
 test_class().getInputProcessor
 )
+if not test_class.inputs:
+test_class.inputs = [None]
 
 def generateMeasurements(self):
 """
diff --git a/ooni/nettests/blocking/web_connectivity.py 
b/ooni/nettests/blocking/web_connectivity.py
index bc23f6d..8d83ad9 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -3,6 +3,8 @@
 import json
 from urlparse import urlparse
 
+from ipaddr import IPv4Address, AddressValueError
+
 from twisted.internet import reactor
 from twisted.internet.protocol import Factory, Protocol
 from twisted.internet.endpoints import TCP4ClientEndpoint
@@ -10,6 +12,7 @@ from twisted.internet.endpoints import TCP4ClientEndpoint
 from twisted.internet import defer
 from twisted.python import usage
 
+from ooni import geoip
 from ooni.utils import log
 
 from ooni.utils.net import StringProducer, BodyReceiver
@@ -34,6 +37,16 @@ class UsageOptions(usage.Options):
 ]
 
 
+def is_public_ipv4_address(address):
+try:
+ip_address = IPv4Address(address)
+if not any([ip_address.is_private,
+ip_address.is_loopback]):
+return True
+return False
+except AddressValueError:
+return None
+
 class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 """
 Web connectivity
@@ -51,8 +64,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 ]
 
 requiredTestHelpers = {
-'backend': 'web_connectivity',
-'dns-discovery': 'dns_discovery'
+'backend': 'web-connectivity',
+'dns-discovery': 'dns-discovery'
 }
 requiresRoot = False
 requiresTor = False
@@ -78,8 +91,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 self.report['control_failure'] = None
 self.report['experiment_failure'] = None
 
-self.report['tcp_connect'] = [
-]
+self.report['tcp_connect'] = []
+self.report['control'] = {}
 
 self.hostname = urlparse(self.input).netloc
 if not self.hostname:
@@ -147,6 +160,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 response.deliverBody(BodyReceiver(finished, content_length))
 body = yield finished
 self.control = json.loads(body)
+self.report['control'] = self.control
 
 def experiment_http_get_request(self):
 return self.doRequest(self.input)
@@ -168,33 +182,48 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 self.report['body_proportion'] = rel
 if rel > float(self.factor):
 self.report['body_length_match'] = True
-return None
+return True
 else:
 self.report['body_length_match'] = False
-return 'http'
+return False
 
 def compare_dns_experiments(self, experiment_dns_answers):
 control_ips = set(self.control['dns']['ips'])
 experiment_ips = set(experiment_dns_answers)
 
+for experiment_ip in experiment_ips:
+if 

[tor-commits] [ooni-probe/master] Fix report uploader to work with JSON

[art]

commit 613a02cdfe44c14fc8a1e91003a8617d913f
Author: Arturo Filastò 
Date:   Thu Feb 4 16:39:14 2016 +0100

Fix report uploader to work with JSON

* Add better exception handling to deckgen
* We should actually be using POST since PUT is indempotent
* Debugging of backward and forward data format compatibility
* Better integrity checks in web_consistency
---
 ooni/nettests/blocking/web_connectivity.py | 4 +++-
 ooni/reporter.py   | 2 --
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/ooni/nettests/blocking/web_connectivity.py 
b/ooni/nettests/blocking/web_connectivity.py
index a61819a..6734618 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -106,6 +106,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 },
 'http_request': {
 'body_length': None,
+'failure': True,
 'headers': {}
 }
 }
@@ -238,7 +239,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 tcp_connect = None
 
 if self.report['control_failure'] is None and \
-self.report['http_experiment_failure'] is None:
+self.report['http_experiment_failure'] is None and \
+self.report['control']['http_request']['failure'] is None:
 body_length_match = 
self.compare_body_lengths(experiment_http_response)
 
 if self.report['control_failure'] is None:
diff --git a/ooni/reporter.py b/ooni/reporter.py
index 49df443..4318fd5 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -201,7 +201,6 @@ class YAMLReporter(OReporter):
 raise Exception("Failed to serialise entry")
 content += safe_dump(report_entry)
 content += '...\n'
-report_entry.update(self.testDetails)
 self._write(content)
 
 def createReport(self):
@@ -295,7 +294,6 @@ class OONIBReporter(OReporter):
 report_entry = entry
 else:
 raise Exception("Failed to serialise entry")
-report_entry.update(self.testDetails)
 content += safe_dump(report_entry)
 content += '...\n'
 return content



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [translation/tails-openpgp-applet] Update translations for tails-openpgp-applet

[translation]

commit c092e33521ca7ff1dab3ea7a35b89d212d87694b
Author: Translation commit bot 
Date:   Mon May 30 14:17:05 2016 +

Update translations for tails-openpgp-applet
---
 et/openpgp-applet.pot | 7 ---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/et/openpgp-applet.pot b/et/openpgp-applet.pot
index b3178a6..32e50ca 100644
--- a/et/openpgp-applet.pot
+++ b/et/openpgp-applet.pot
@@ -3,13 +3,14 @@
 # This file is distributed under the same license as the PACKAGE package.
 # 
 # Translators:
+# Kristjan Räts , 2016
 msgid ""
 msgstr ""
 "Project-Id-Version: The Tor Project\n"
 "Report-Msgid-Bugs-To: ta...@boum.org\n"
 "POT-Creation-Date: 2015-08-10 15:55+0200\n"
-"PO-Revision-Date: 2015-11-23 02:23+\n"
-"Last-Translator: FULL NAME \n"
+"PO-Revision-Date: 2016-05-30 13:49+\n"
+"Last-Translator: Kristjan Räts \n"
 "Language-Team: Estonian 
(http://www.transifex.com/otf/torproject/language/et/)\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
@@ -43,7 +44,7 @@ msgstr ""
 
 #: bin/openpgp-applet:243
 msgid "_Decrypt/Verify Clipboard"
-msgstr ""
+msgstr "_Dekrüpteeri/Kontrolli lõikepuhvrit"
 
 #: bin/openpgp-applet:247
 msgid "_Manage Keys"

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [translation/torbutton-abouttorproperties_completed] Update translations for torbutton-abouttorproperties_completed

[translation]

commit 825a612ee4c784e6dd652c9b60fff087baa569fe
Author: Translation commit bot 
Date:   Mon May 30 14:16:28 2016 +

Update translations for torbutton-abouttorproperties_completed
---
 et/abouttor.properties | 21 +
 1 file changed, 21 insertions(+)

diff --git a/et/abouttor.properties b/et/abouttor.properties
new file mode 100644
index 000..dbbeb4a
--- /dev/null
+++ b/et/abouttor.properties
@@ -0,0 +1,21 @@
+# Copyright (c) 2014, The Tor Project, Inc.
+# See LICENSE for licensing information.
+# vim: set sw=2 sts=2 ts=8 et:
+
+aboutTor.searchSP.privacy=Otsi turvaliselt avalehega.
+# The following string is a link which replaces %1$S above.
+aboutTor.searchSP.privacy.link=https://startpage.com/eng/protect-privacy.html
+# The following string is a link which replaces %2$S above.
+aboutTor.searchSP.search.link=https://startpage.com/
+
+aboutTor.searchDDG.privacy=Otsi turvaliselt DickDuckGoga.
+# The following string is a link which replaces %1$S above.
+aboutTor.searchDDG.privacy.link=https://duckduckgo.com/privacy.html
+# The following string is a link which replaces %2$S above.
+aboutTor.searchDDG.search.link=https://duckduckgo.com/
+
+aboutTor.searchDC.privacy=Otsi turvaliselt Disconnect.mega.
+# The following string is a link which replaces %1$S above.
+aboutTor.searchDC.privacy.link=https://disconnect.me/privacy
+# The following string is a link which replaces %2$S above.
+aboutTor.searchDC.search.link=https://search.disconnect.me/

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [translation/torbutton-aboutdialogdtd] Update translations for torbutton-aboutdialogdtd

[translation]

commit 66550db622de8575b07dfaf2d93e9801102d87c3
Author: Translation commit bot 
Date:   Mon May 30 14:16:18 2016 +

Update translations for torbutton-aboutdialogdtd
---
 et/aboutdialog.dtd | 22 +++---
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/et/aboutdialog.dtd b/et/aboutdialog.dtd
index 5099ad7..98fb292 100644
--- a/et/aboutdialog.dtd
+++ b/et/aboutdialog.dtd
@@ -1,19 +1,19 @@
-
+
 
-
-
+
+
 
-
+
 
-
-
+
+
 
-
+
 
 
-
+
 
-
+
 
-
-
+
+

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [translation/tails-perl5lib_completed] Update translations for tails-perl5lib_completed

[translation]

commit e4dbcd36d52f3449339eb4e857feb0299466f198
Author: Translation commit bot 
Date:   Mon May 30 14:16:11 2016 +

Update translations for tails-perl5lib_completed
---
 et.po | 35 +++
 1 file changed, 35 insertions(+)

diff --git a/et.po b/et.po
new file mode 100644
index 000..96a92c9
--- /dev/null
+++ b/et.po
@@ -0,0 +1,35 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR Tails developers
+# This file is distributed under the same license as the PACKAGE package.
+# 
+# Translators:
+# Kristjan Räts , 2016
+msgid ""
+msgstr ""
+"Project-Id-Version: The Tor Project\n"
+"Report-Msgid-Bugs-To: Tails developers \n"
+"POT-Creation-Date: 2016-04-24 16:36+0200\n"
+"PO-Revision-Date: 2016-05-30 14:13+\n"
+"Last-Translator: Kristjan Räts \n"
+"Language-Team: Estonian 
(http://www.transifex.com/otf/torproject/language/et/)\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Language: et\n"
+"Plural-Forms: nplurals=2; plural=(n != 1);\n"
+
+#: ../lib/Tails/RunningSystem.pm:159
+msgid "Error"
+msgstr "Viga"
+
+#: ../lib/Tails/RunningSystem.pm:161
+msgid ""
+"The device Tails is running from cannot be found. Maybe you used the `toram'"
+" option?"
+msgstr "Seade, millel Tails käivitati, ei ole leitav. Võib-olla kasutasid 
parameetrit 'toram'?"
+
+#: ../lib/Tails/RunningSystem.pm:192
+msgid ""
+"The drive Tails is running from cannot be found. Maybe you used the `toram' "
+"option?"
+msgstr "Ketas, millel Tails käivitati, ei ole leitav. Võib-olla kasutasid 
parameetrit 'toram'?"

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [translation/torbutton-aboutdialogdtd_completed] Update translations for torbutton-aboutdialogdtd_completed

[translation]

commit dda3e8bf32f849933e877b411973a80775245da4
Author: Translation commit bot 
Date:   Mon May 30 14:16:21 2016 +

Update translations for torbutton-aboutdialogdtd_completed
---
 et/aboutdialog.dtd | 19 +++
 1 file changed, 19 insertions(+)

diff --git a/et/aboutdialog.dtd b/et/aboutdialog.dtd
new file mode 100644
index 000..98fb292
--- /dev/null
+++ b/et/aboutdialog.dtd
@@ -0,0 +1,19 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [translation/torbutton-abouttorproperties] Update translations for torbutton-abouttorproperties

[translation]

commit 35bfc1bb48919403b084a6cb052d801d0238adef
Author: Translation commit bot 
Date:   Mon May 30 14:16:25 2016 +

Update translations for torbutton-abouttorproperties
---
 et/abouttor.properties | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/et/abouttor.properties b/et/abouttor.properties
index d607324..dbbeb4a 100644
--- a/et/abouttor.properties
+++ b/et/abouttor.properties
@@ -2,19 +2,19 @@
 # See LICENSE for licensing information.
 # vim: set sw=2 sts=2 ts=8 et:
 
-aboutTor.searchSP.privacy=Search securely with Startpage.
+aboutTor.searchSP.privacy=Otsi turvaliselt avalehega.
 # The following string is a link which replaces %1$S above.
 aboutTor.searchSP.privacy.link=https://startpage.com/eng/protect-privacy.html
 # The following string is a link which replaces %2$S above.
 aboutTor.searchSP.search.link=https://startpage.com/
 
-aboutTor.searchDDG.privacy=Search securely with DuckDuckGo.
+aboutTor.searchDDG.privacy=Otsi turvaliselt DickDuckGoga.
 # The following string is a link which replaces %1$S above.
 aboutTor.searchDDG.privacy.link=https://duckduckgo.com/privacy.html
 # The following string is a link which replaces %2$S above.
 aboutTor.searchDDG.search.link=https://duckduckgo.com/
 
-aboutTor.searchDC.privacy=Search securely with Disconnect.me.
+aboutTor.searchDC.privacy=Otsi turvaliselt Disconnect.mega.
 # The following string is a link which replaces %1$S above.
 aboutTor.searchDC.privacy.link=https://disconnect.me/privacy
 # The following string is a link which replaces %2$S above.

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [translation/tails-perl5lib] Update translations for tails-perl5lib

[translation]

commit e870383958daecb8ace4f302515df779ea419d86
Author: Translation commit bot 
Date:   Mon May 30 14:16:07 2016 +

Update translations for tails-perl5lib
---
 et.po | 13 +++--
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/et.po b/et.po
index 6d4835a..96a92c9 100644
--- a/et.po
+++ b/et.po
@@ -3,13 +3,14 @@
 # This file is distributed under the same license as the PACKAGE package.
 # 
 # Translators:
+# Kristjan Räts , 2016
 msgid ""
 msgstr ""
 "Project-Id-Version: The Tor Project\n"
 "Report-Msgid-Bugs-To: Tails developers \n"
-"POT-Creation-Date: 2016-01-11 15:40+0100\n"
-"PO-Revision-Date: 2016-01-12 08:13+\n"
-"Last-Translator: carolyn \n"
+"POT-Creation-Date: 2016-04-24 16:36+0200\n"
+"PO-Revision-Date: 2016-05-30 14:13+\n"
+"Last-Translator: Kristjan Räts \n"
 "Language-Team: Estonian 
(http://www.transifex.com/otf/torproject/language/et/)\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
@@ -19,16 +20,16 @@ msgstr ""
 
 #: ../lib/Tails/RunningSystem.pm:159
 msgid "Error"
-msgstr ""
+msgstr "Viga"
 
 #: ../lib/Tails/RunningSystem.pm:161
 msgid ""
 "The device Tails is running from cannot be found. Maybe you used the `toram'"
 " option?"
-msgstr ""
+msgstr "Seade, millel Tails käivitati, ei ole leitav. Võib-olla kasutasid 
parameetrit 'toram'?"
 
 #: ../lib/Tails/RunningSystem.pm:192
 msgid ""
 "The drive Tails is running from cannot be found. Maybe you used the `toram' "
 "option?"
-msgstr ""
+msgstr "Ketas, millel Tails käivitati, ei ole leitav. Võib-olla kasutasid 
parameetrit 'toram'?"

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [translation/tails-misc] Update translations for tails-misc

[translation]

commit 292091ee6da5bedc5c200952e9c30bc4709e7ee3
Author: Translation commit bot 
Date:   Mon May 30 14:15:51 2016 +

Update translations for tails-misc
---
 et.po | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/et.po b/et.po
index 273a531..e744b2e 100644
--- a/et.po
+++ b/et.po
@@ -8,7 +8,7 @@ msgstr ""
 "Project-Id-Version: The Tor Project\n"
 "Report-Msgid-Bugs-To: \n"
 "POT-Creation-Date: 2016-05-25 20:25+0200\n"
-"PO-Revision-Date: 2016-05-30 13:36+\n"
+"PO-Revision-Date: 2016-05-30 14:11+\n"
 "Last-Translator: carolyn \n"
 "Language-Team: Estonian 
(http://www.transifex.com/otf/torproject/language/et/)\n"
 "MIME-Version: 1.0\n"
@@ -91,7 +91,7 @@ msgstr ""
 
 #: config/chroot_local-includes/usr/local/bin/gpgApplet:200
 msgid "_Decrypt/Verify Clipboard"
-msgstr ""
+msgstr "_Dekrüpteeri/Kontrolli lõikepuhvrit"
 
 #: config/chroot_local-includes/usr/local/bin/gpgApplet:204
 msgid "_Manage Keys"
@@ -348,7 +348,7 @@ msgstr ""
 #: config/chroot_local-includes/usr/local/bin/tails-upgrade-frontend-wrapper:25
 #: config/chroot_local-includes/usr/local/sbin/unsafe-browser:28
 msgid "Error"
-msgstr ""
+msgstr "Viga"
 
 #: config/chroot_local-includes/usr/local/bin/tails-upgrade-frontend-wrapper:45
 msgid ""

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [translation/tails-persistence-setup] Update translations for tails-persistence-setup

[translation]

commit 37faefc09f83c54c7695cae255e33a34427f73ad
Author: Translation commit bot 
Date:   Mon May 30 14:15:27 2016 +

Update translations for tails-persistence-setup
---
 et/et.po | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/et/et.po b/et/et.po
index d86b84e..abe4b30 100644
--- a/et/et.po
+++ b/et/et.po
@@ -9,7 +9,7 @@ msgstr ""
 "Project-Id-Version: The Tor Project\n"
 "Report-Msgid-Bugs-To: Tails developers \n"
 "POT-Creation-Date: 2016-04-24 16:40+0200\n"
-"PO-Revision-Date: 2016-05-30 13:45+\n"
+"PO-Revision-Date: 2016-05-30 14:11+\n"
 "Last-Translator: Kristjan Räts \n"
 "Language-Team: Estonian 
(http://www.transifex.com/otf/torproject/language/et/)\n"
 "MIME-Version: 1.0\n"
@@ -137,7 +137,7 @@ msgstr ""
 
 #: ../lib/Tails/Persistence/Setup.pm:312 ../lib/Tails/Persistence/Setup.pm:459
 msgid "Error"
-msgstr ""
+msgstr "Viga"
 
 #: ../lib/Tails/Persistence/Setup.pm:344
 #, perl-format

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [translation/https_everywhere] Update translations for https_everywhere

[translation]

commit f682ec3a3fce8d050397117e326f6842c5545e76
Author: Translation commit bot 
Date:   Mon May 30 14:15:15 2016 +

Update translations for https_everywhere
---
 et/ssl-observatory.dtd | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/et/ssl-observatory.dtd b/et/ssl-observatory.dtd
index da75df6..dd033be 100644
--- a/et/ssl-observatory.dtd
+++ b/et/ssl-observatory.dtd
@@ -20,7 +20,7 @@ to turn it on?">-->
 
 
 
 
 ___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [translation/https_everywhere_completed] Update translations for https_everywhere_completed

[translation]

commit 922df1443011a7f284b28053b924ec7a1bfb3837
Author: Translation commit bot 
Date:   Mon May 30 14:15:21 2016 +

Update translations for https_everywhere_completed
---
 et/ssl-observatory.dtd | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/et/ssl-observatory.dtd b/et/ssl-observatory.dtd
index da75df6..dd033be 100644
--- a/et/ssl-observatory.dtd
+++ b/et/ssl-observatory.dtd
@@ -20,7 +20,7 @@ to turn it on?">-->
 
 
 
 
 ___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [translation/torbirdy_completed] Update translations for torbirdy_completed

[translation]

commit 6d9b0dee869dfbec44e1b518ae81afb3c7912a2d
Author: Translation commit bot 
Date:   Mon May 30 14:15:38 2016 +

Update translations for torbirdy_completed
---
 et/torbirdy.properties | 18 ++
 1 file changed, 18 insertions(+)

diff --git a/et/torbirdy.properties b/et/torbirdy.properties
new file mode 100644
index 000..f91bb8f
--- /dev/null
+++ b/et/torbirdy.properties
@@ -0,0 +1,18 @@
+torbirdy.name=TorBirdy
+
+torbirdy.enabled.tor=TorBirdy aktiivne: Tor
+torbirdy.enabled.jondo=TorBirdy aktiivne: JonDo
+torbirdy.enabled.custom=TorBirdy aktiivne: kohandatud proksi
+torbirdy.enabled.torification=TorBirdy aktiivne: läbipaistev Tor'indamine
+torbirdy.enabled.whonix=TorBirdy aktiivne: Whonix
+torbirdy.disabled=TorBirdy: mitteaktiivne!
+torbirdy.enabled=TorBirdy: aktiivne
+
+torbirdy.email.prompt=TorBirdy keelas Thunderbirdi automaathäälestuse 
visardi, et kaitsta su anonüümsust.\n\nSoovitatavad turvalisuse seaded on 
%S-le määratud.\n\nSa saad konto ülejäänud seaded häälestada käsitsi.
+
+torbirdy.email.advanced=Palun aevesta, et TorBirdy täpsemate seadete muutmine 
ei ole soovitatav.\n\nSa peaksid jätkama ainult siis, kui oled kindel selles, 
mida sa teed.
+torbirdy.email.advanced.nextwarning=Kuva seda hoiatust ka järgmisel korral
+torbirdy.email.advanced.title=TorBirdy täpsemad seaded
+
+torbirdy.firstrun=Sa kastad nüüd TorBirdy't.\n\nSinu anonüümsuse 
tagamiseks muudab ta Thunderbirdy sätted, mis ta määras, sinu või teiste 
laiendite poolt mittemuudetavateks. Mõningaid sätteid saab muuta ja selleks 
kasuta TorBirdy seadete dialoogi. Kui TorBirdy eemaldatakse või 
deaktiveeritakse, siis lähtestatakse need seaded nende TorBirdy paigaldamise 
eelsetele väärtustele.\n\nKui sa oled uus kasutaja, siis on soovitatav, et sa 
loeksid käbi TorBirdy veebilehe, et saada aru, mida me proovime oma 
kasutajatele TorBirdyga pakkuda.
+torbirdy.website=https://trac.torproject.org/projects/tor/wiki/torbirdy

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [translation/tor-messenger-privdtd] Update translations for tor-messenger-privdtd

[translation]

commit 37dfd6b3ce63caa24985c286abcacf3a6f7e0847
Author: Translation commit bot 
Date:   Mon May 30 13:46:53 2016 +

Update translations for tor-messenger-privdtd
---
 et/priv.dtd | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/et/priv.dtd b/et/priv.dtd
index 2e65400..c62bb33 100644
--- a/et/priv.dtd
+++ b/et/priv.dtd
@@ -1,2 +1,2 @@
-
+
 
\ No newline at end of file

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [translation/torbutton-branddtd_completed] Update translations for torbutton-branddtd_completed

[translation]

commit 5041bb375d174c41d3007ccdff897a73efc1e265
Author: Translation commit bot 
Date:   Mon May 30 13:46:02 2016 +

Update translations for torbutton-branddtd_completed
---
 et/brand.dtd | 15 +++
 1 file changed, 15 insertions(+)

diff --git a/et/brand.dtd b/et/brand.dtd
new file mode 100644
index 000..a47c121
--- /dev/null
+++ b/et/brand.dtd
@@ -0,0 +1,15 @@
+
+
+
+
+
+
+
+
+
+
+
+
+

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [translation/torbutton-torbuttonproperties] Update translations for torbutton-torbuttonproperties

[translation]

commit b53ae43b31c9c8add62e1eae5348d0cfb0c572cd
Author: Translation commit bot 
Date:   Mon May 30 13:46:05 2016 +

Update translations for torbutton-torbuttonproperties
---
 et/torbutton.properties | 18 +-
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/et/torbutton.properties b/et/torbutton.properties
index 51e1568..0e728a3 100644
--- a/et/torbutton.properties
+++ b/et/torbutton.properties
@@ -42,17 +42,17 @@ torbutton.popup.captcha.ask = Tor tuvastas Google CAPTCHA. 
Kas sa soovid, et sin
 torbutton.popup.captcha.always = Nüüdsest edasi soorita alati see tegevus
 torbutton.popup.redirect = Suuna ümber
 torbutton.popup.no_redirect = Ära suuna ümber
-torbutton.popup.prompted_language = To give you more privacy, Torbutton can 
request the English language version of web pages. This may cause web pages 
that you prefer to read in your native language to display in English 
instead.\n\nWould you like to request English language web pages for better 
privacy?
-torbutton.popup.no_newnym = Torbutton cannot safely give you a new identity. 
It does not have access to the Tor Control Port.\n\nAre you running Tor Browser 
Bundle?
-torbutton.popup.use_tbb = Näib, et sa kasutat Torbuttonit koos Firefoxiga, 
mis pole enam soovitatud turvaline lahendus.\n\nMe soovitame sul selle asemel 
hankida Tori Veebilehitseja Pakett saates e-posti aadressile 
get...@torproject.org või laadides selle alla järgnevalt aadressilt:
-torbutton.popup.pref_error = Torbutton cannot update preferences in the Tor 
Browser profile directory.
-torbutton.popup.permission_denied = Please either reset the permissions of the 
Tor Browser directory or copy it to a new location.
-torbutton.popup.device_full = The disk appears to be full. Please free up 
space or move Tor Browser to a new device.
+torbutton.popup.prompted_language = Täiendava privaatsuse andmiseks võib 
Torbutton küsida veebilehe inglise keelset versiooni. Selle tulemusel 
võidakse sulle kuvada lehti, mida loed hetkel oma keeles hoopis inglise 
keeles.\n\nKas sa soovid parema privaatsuse nimel küsida inglise keelseid 
veebilehti?
+torbutton.popup.no_newnym = Torbutton ei saa sulle turvaliselt anda uut 
identiteeti. Tal puudub pilipääs Tor'i juhtpordile.\n\nKas sa kasutad Tor 
Brauseri paketti?
+torbutton.popup.use_tbb = Näib, et sa kasutat Torbuttonit koos Firefoxiga, 
mis pole enam soovitatud turvaline lahendus.\n\nMe soovitame sul selle asemel 
hankida Tor'i Browser paketi saates e-kirja aadressile get...@torproject.org 
või laadides selle alla järgnevalt aadressilt:
+torbutton.popup.pref_error = Torbutton ei saa uuendada eelistusi Tor Brauseri 
profiili kataloogis.
+torbutton.popup.permission_denied = Palun lähtesta Tor Brauseri kataloogi 
õigused või kopeeri kataloog uude asukohta.
+torbutton.popup.device_full = Ketas tundub olevat täis. Palun tee rohkem vaba 
ruumi või liiguta Tor Brauser uuele seadmele.
 torbutton.title.prompt_torbrowser = Oluline Torbuttoni informatsioon
-torbutton.popup.prompt_torbrowser = Torbutton works differently now: you can't 
turn it off any more.\n\nWe made this change because it isn't safe to use 
Torbutton in a browser that's also used for non-Tor browsing. There were too 
many bugs there that we couldn't fix any other way.\n\nIf you want to keep 
using Firefox normally, you should uninstall Torbutton and download Tor Browser 
Bundle. The privacy properties of Tor Browser are also superior to those of 
normal Firefox, even when Firefox is used with Torbutton.\n\nTo remove 
Torbutton, go to Tools->Addons->Extensions and then click the Remove button 
next to Torbutton.
-torbutton.popup.short_torbrowser = Important Torbutton 
Information!\n\nTorbutton is now always enabled.\n\nClick on the Torbutton for 
more information.
+torbutton.popup.prompt_torbrowser = Torbutton töötab nüüd erinevalt: sa ei 
saa teda enam välja lülitada.\n\nMe muutsime seda, sest Torbuttoni kasutamine 
brauseris, mida kasutatakse ka Tor võrgust väljas, ei ole turvaline. Sellega 
kaasnes liiga palju vigu, mida ei saanud lahendada muud moodi.\n\nKui sa soovid 
kasutada Firefoxi tavapäraselt, siis eemalda Torbutton ja laadi alla Tor 
Brauseri pakett. Tor Brauseri privaatsus on samuti parem kui Firefoxil, isegi 
koos Torbuttoniga.\n\nTorbuttoni eemaldamiseks mine 
Tööriistad->Lisad->Laiendused ja klõpsa Torbuttoni juures nupul Eemalda
+torbutton.popup.short_torbrowser = Tähtis Torbutton'i teave!\n\nTorbutton on 
nüüd alati lubatud.\n\nLisainfo saamiseks klõpsa Torbuttonil.
 
-torbutton.popup.confirm_plugins = Plugins such as Flash can harm your privacy 
and anonymity.\n\nThey can also bypass Tor to reveal your current location and 
IP address.\n\nAre you sure you want to enable plugins?\n\n
+torbutton.popup.confirm_plugins = Pistikprogrammid nagu Flash võivad 
kahjustada su privaatsust ja anonüümsust.\n\nNad saavad Tor'ist mööda minna 
ja paljastada su praeguse asukoha ja 

[tor-commits] [translation/tor-messenger-privdtd_completed] Update translations for tor-messenger-privdtd_completed

[translation]

commit 126dbff14ac9041c5e6f5ea189a4c3d7d666ce01
Author: Translation commit bot 
Date:   Mon May 30 13:46:56 2016 +

Update translations for tor-messenger-privdtd_completed
---
 et/priv.dtd | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/et/priv.dtd b/et/priv.dtd
new file mode 100644
index 000..c62bb33
--- /dev/null
+++ b/et/priv.dtd
@@ -0,0 +1,2 @@
+
+
\ No newline at end of file

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [translation/gettor] Update translations for gettor

[translation]

commit 1b42a737f2cea0cc4b0d0e2ef27f2a1e9b694181
Author: Translation commit bot 
Date:   Mon May 30 13:45:06 2016 +

Update translations for gettor
---
 et/gettor.po | 502 +++
 1 file changed, 502 insertions(+)

diff --git a/et/gettor.po b/et/gettor.po
new file mode 100644
index 000..f97c3af
--- /dev/null
+++ b/et/gettor.po
@@ -0,0 +1,502 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
+# This file is distributed under the same license as the PACKAGE package.
+# 
+# Translators:
+msgid ""
+msgstr ""
+"Project-Id-Version: The Tor Project\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2013-01-19 13:40+0100\n"
+"PO-Revision-Date: 2010-11-30 05:01+\n"
+"Last-Translator: FULL NAME \n"
+"Language-Team: Estonian 
(http://www.transifex.com/otf/torproject/language/et/)\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Language: et\n"
+"Plural-Forms: nplurals=2; plural=(n != 1);\n"
+
+#: lib/gettor/i18n.py:27
+msgid "Hello, This is the \"GetTor\" robot."
+msgstr ""
+
+#: lib/gettor/i18n.py:29
+msgid "Thank you for your request."
+msgstr ""
+
+#: lib/gettor/i18n.py:31
+msgid ""
+"Unfortunately, we won't answer you at this address. You should make\n"
+"an account with GMAIL.COM, YAHOO.COM or YAHOO.CN and send the mail from\n"
+"one of those."
+msgstr ""
+
+#: lib/gettor/i18n.py:35
+msgid ""
+"We only process requests from email services that support \"DKIM\",\n"
+"which is an email feature that lets us verify that the address in the\n"
+"\"From\" line is actually the one who sent the mail."
+msgstr ""
+
+#: lib/gettor/i18n.py:39
+msgid ""
+"(We apologize if you didn't ask for this mail. Since your email is from\n"
+"a service that doesn't use DKIM, we're sending a short explanation,\n"
+"and then we'll ignore this email address for the next day or so.)"
+msgstr ""
+
+#: lib/gettor/i18n.py:43 lib/gettor/i18n.py:135
+msgid ""
+"If you have any questions or it doesn't work, you can contact a\n"
+"human at this support email address: h...@rt.torproject.org"
+msgstr ""
+
+#: lib/gettor/i18n.py:46
+msgid ""
+"I will mail you a Tor package, if you tell me which one you want.\n"
+"Please select one of the following package names:\n"
+"\n"
+"windows\n"
+"macos-i386\n"
+"macos-ppc\n"
+"linux-i386\n"
+"linux-x86_64\n"
+"obfs-windows\n"
+"obfs-macos-i386\n"
+"obfs-macos-x86_64\n"
+"obfs-linux-i386\n"
+"obfs-linux-x86_64\n"
+"source"
+msgstr ""
+
+#: lib/gettor/i18n.py:61
+msgid ""
+"Please reply to this mail, and tell me a single package name anywhere \n"
+"in the body of your email."
+msgstr ""
+
+#: lib/gettor/i18n.py:64
+msgid ""
+"OBTAINING LOCALIZED VERSIONS OF TOR\n"
+"==="
+msgstr ""
+
+#: lib/gettor/i18n.py:67
+msgid ""
+"To get a version of Tor translated into your language, specify the\n"
+"language you want in the address you send the mail to:\n"
+"\n"
+"gettor...@torproject.org"
+msgstr ""
+
+#: lib/gettor/i18n.py:72
+msgid ""
+"This example will give you the requested package in a localized\n"
+"version for Farsi (Persian). Check below for a list of supported language\n"
+"codes. "
+msgstr ""
+
+#: lib/gettor/i18n.py:76
+msgid " List of supported locales:"
+msgstr ""
+
+#: lib/gettor/i18n.py:78
+msgid "Here is a list of all available languages:"
+msgstr ""
+
+#: lib/gettor/i18n.py:80
+msgid ""
+"gettor...@torproject.org: Arabic\n"
+"gettor...@torproject.org: German\n"
+"gettor...@torproject.org: English\n"
+"gettor...@torproject.org: Spanish\n"
+"gettor...@torproject.org: Farsi (Iran)\n"
+"gettor...@torproject.org: French\n"
+"gettor...@torproject.org: Italian\n"
+"gettor...@torproject.org: Dutch\n"
+"gettor...@torproject.org: Polish\n"
+"gettor...@torproject.org: Russian\n"
+"gettor...@torproject.org: Chinese"
+msgstr ""
+
+#: lib/gettor/i18n.py:92
+msgid "If you select no language, you will receive the English version."
+msgstr ""
+
+#: lib/gettor/i18n.py:94
+msgid ""
+"SMALLER SIZED PACKAGES\n"
+"=="
+msgstr ""
+
+#: lib/gettor/i18n.py:97
+msgid ""
+"If your bandwith is low or your provider doesn't allow you to\n"
+"receive large attachments in your email, GetTor can send you several\n"
+"small packages instead of one big one."
+msgstr ""
+
+#: lib/gettor/i18n.py:101
+msgid ""
+"Simply include the keyword 'split' in a new line on its own (this part\n"
+"is important!) like so: \n"
+"\n"
+"windows\n"
+"split"
+msgstr ""
+
+#: lib/gettor/i18n.py:107
+msgid ""
+"Sending this text in an email to GetTor will cause it to send you \n"
+"the Tor Browser Bundle in a number of 1,4MB attachments."
+msgstr ""
+
+#: lib/gettor/i18n.py:110
+msgid ""
+"After having received all parts, you need to re-assemble them to 

[tor-commits] [translation/torbutton-brandproperties_completed] Update translations for torbutton-brandproperties_completed

[translation]

commit cb4b56e13693e9c943e8d309589939ef54fc652e
Author: Translation commit bot 
Date:   Mon May 30 13:45:55 2016 +

Update translations for torbutton-brandproperties_completed
---
 et/brand.properties | 16 
 1 file changed, 16 insertions(+)

diff --git a/et/brand.properties b/et/brand.properties
new file mode 100644
index 000..c40974b
--- /dev/null
+++ b/et/brand.properties
@@ -0,0 +1,16 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+brandShorterName=Tor Brauser
+brandShortName=Tor Brauser
+brandFullName=Tor Brauser
+vendorShortName=Projekt Tor
+
+homePageSingleStartMain=Firefox Start, kiire koduleht sisseehitatud otsinguga
+homePageImport=Impordi oma koduleht %S-st
+
+homePageMigrationPageTitle=Kodulehe valimine
+homePageMigrationDescription=Palun vali koduleht, mida sa soovid kasutada:
+
+syncBrandShortName=Sünkroniseeri

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [translation/torbutton-branddtd] Update translations for torbutton-branddtd

[translation]

commit fbb6420217d6ca893597e5f5661cf87ec3660528
Author: Translation commit bot 
Date:   Mon May 30 13:45:59 2016 +

Update translations for torbutton-branddtd
---
 et/brand.dtd | 18 +-
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/et/brand.dtd b/et/brand.dtd
index 3df1a08..a47c121 100644
--- a/et/brand.dtd
+++ b/et/brand.dtd
@@ -2,14 +2,14 @@
- License, v. 2.0. If a copy of the MPL was not distributed with this
- file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
 
-
-
-
-
-
+
+
+
+
+
 
 
-
-
-
-
+
+
+
+

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [translation/tails-misc] Update translations for tails-misc

[translation]

commit b880ec49547b6602a4440cfcb0ff6e45edd1a323
Author: Translation commit bot 
Date:   Mon May 30 13:45:45 2016 +

Update translations for tails-misc
---
 et.po | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/et.po b/et.po
index 6746ff2..273a531 100644
--- a/et.po
+++ b/et.po
@@ -8,7 +8,7 @@ msgstr ""
 "Project-Id-Version: The Tor Project\n"
 "Report-Msgid-Bugs-To: \n"
 "POT-Creation-Date: 2016-05-25 20:25+0200\n"
-"PO-Revision-Date: 2016-05-27 10:12+\n"
+"PO-Revision-Date: 2016-05-30 13:36+\n"
 "Last-Translator: carolyn \n"
 "Language-Team: Estonian 
(http://www.transifex.com/otf/torproject/language/et/)\n"
 "MIME-Version: 1.0\n"
@@ -537,7 +537,7 @@ msgstr ""
 
 #: 
../config/chroot_local-includes/usr/share/applications/tor-browser.desktop.in.h:1
 msgid "Tor Browser"
-msgstr ""
+msgstr "Tor Brauser"
 
 #: 
../config/chroot_local-includes/usr/share/applications/tor-browser.desktop.in.h:2
 msgid "Anonymous Web Browser"

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [translation/tails-persistence-setup] Update translations for tails-persistence-setup

[translation]

commit 4e0fdf41d25c607b8c7e5cfd9b150a072ef63e45
Author: Translation commit bot 
Date:   Mon May 30 13:45:24 2016 +

Update translations for tails-persistence-setup
---
 et/et.po | 7 ---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/et/et.po b/et/et.po
index 40dfda8..d86b84e 100644
--- a/et/et.po
+++ b/et/et.po
@@ -3,13 +3,14 @@
 # This file is distributed under the same license as the PACKAGE package.
 # 
 # Translators:
+# Kristjan Räts , 2016
 msgid ""
 msgstr ""
 "Project-Id-Version: The Tor Project\n"
 "Report-Msgid-Bugs-To: Tails developers \n"
 "POT-Creation-Date: 2016-04-24 16:40+0200\n"
-"PO-Revision-Date: 2016-04-25 08:33+\n"
-"Last-Translator: carolyn \n"
+"PO-Revision-Date: 2016-05-30 13:45+\n"
+"Last-Translator: Kristjan Räts \n"
 "Language-Team: Estonian 
(http://www.transifex.com/otf/torproject/language/et/)\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
@@ -39,7 +40,7 @@ msgstr ""
 
 #: ../lib/Tails/Persistence/Configuration/Presets.pm:70
 msgid "SSH keys, configuration and known hosts"
-msgstr ""
+msgstr "SSH võtmed, häälestus ja tuntud hostid"
 
 #: ../lib/Tails/Persistence/Configuration/Presets.pm:78
 msgid "Pidgin"

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [translation/torbutton-brandproperties] Update translations for torbutton-brandproperties

[translation]

commit 5813b520423b0aa11deeb660d82acb4084f04918
Author: Translation commit bot 
Date:   Mon May 30 13:45:52 2016 +

Update translations for torbutton-brandproperties
---
 et/brand.properties | 18 +-
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/et/brand.properties b/et/brand.properties
index 732c157..c40974b 100644
--- a/et/brand.properties
+++ b/et/brand.properties
@@ -2,15 +2,15 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-brandShorterName=Tor Browser
-brandShortName=Tor Browser
-brandFullName=Tor Browser
-vendorShortName=Tor Project
+brandShorterName=Tor Brauser
+brandShortName=Tor Brauser
+brandFullName=Tor Brauser
+vendorShortName=Projekt Tor
 
-homePageSingleStartMain=Firefox Start, a fast home page with built-in search
-homePageImport=Import your home page from %S
+homePageSingleStartMain=Firefox Start, kiire koduleht sisseehitatud otsinguga
+homePageImport=Impordi oma koduleht %S-st
 
-homePageMigrationPageTitle=Home Page Selection
-homePageMigrationDescription=Please select the home page you wish to use:
+homePageMigrationPageTitle=Kodulehe valimine
+homePageMigrationDescription=Palun vali koduleht, mida sa soovid kasutada:
 
-syncBrandShortName=Sync
+syncBrandShortName=Sünkroniseeri

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [oonib/master] Document configuration (#78)

[art]

commit 0396a55a27e9923537a6cd3d743d37009ef4715b
Author: Joe Landers 
Date:   Thu May 26 12:49:23 2016 +0200

Document configuration (#78)

* Update README.rst

* remove old configuration option
---
 README.rst | 17 +
 oonib.conf.example |  1 -
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index 63a242c..a73d776 100644
--- a/README.rst
+++ b/README.rst
@@ -107,6 +107,23 @@ this would simply be ``'report'``) and ``tor_datadir`` 
(where you would
 like the spawned Tor process to keep its data). If you compiled Tor yourself,
 you'll likely want to specify it for the ``tor_binary`` option.
 
+Configure bouncer and collector endpoints
+.
+
+The bouncer and collector are HTTP applications ("protocols" in twisted 
terminology) that can be configured to run on top of plain TCP, TLS, or onion 
service endpoints.
+Here is an example of the relevant part of the configuration::
+bouncer_endpoints:
+- {type: tls, port: 10443, cert: "private/ssl-key-and-cert.pem"}
+- {type: tcp, port: 10080}
+- {type: onion, hsdir: "/some/private/bouncer"}
+
+collector_endpoints:
+- {type: tls, port: 11443, cert: "private/ssl-key-and-cert.pem"}
+- {type: onion, hsdir: "/some/private/collector"}
+
+`scripts/gen-ssl-key-cert.sh` in this repo contains the openssl command to 
generate a self-signed certificate which you can use for the tls endpoint.
+txtorcon will use the hostname/private_key from the configured hsdir to start 
an onion service, or generate a new key if hsdir is empty.
+
 Generate self signed certs for OONIB
 
 If you want to use the HTTPS test helper, you will need to create a
diff --git a/oonib.conf.example b/oonib.conf.example
index 95df448..1925d81 100644
--- a/oonib.conf.example
+++ b/oonib.conf.example
@@ -27,7 +27,6 @@ main:
 debug: false
 stale_time: 3600
 
-tor_hidden_service: false
 tor_datadir: null
 
 bouncer_endpoints:



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [oonib/master] Fix privilege shedding and daemonisation of oonibackend (#80)

[art]

commit 5940999a49af50ff49aeabb1103543889c598312
Author: Arturo Filastò 
Date:   Mon May 30 15:24:07 2016 +0200

Fix privilege shedding and daemonisation of oonibackend (#80)

* Fix privilege shedding and daemonisation of oonibackend

* Rewrite to use twistd

* Launch tor only after we have forked

* Set the UID of when starting tor

This fixes #65

* Add gitignore files and remove empty.txt files

* Drop support for super old versions of txtorcon

* Remove now unnecessary hacks to maintain backward compatibility

* Fix permissions on temporary tor data directory
---
 bin/oonib   |  38 ++-
 data/archive/.gitignore |   2 +
 data/archive/empty.txt  |   2 -
 data/decks/.gitignore   |   3 +
 data/inputs/.gitignore  |   3 +
 data/reports/.gitignore |   2 +
 data/tor/.gitignore |   2 +
 data/tor/empty.txt  |   2 -
 oonib.conf.example  |   4 -
 oonib/config.py |   2 +
 oonib/log.py|  61 +--
 oonib/onion.py  |  63 +--
 oonib/oonibackend.py| 280 ++--
 requirements.txt|   2 +-
 setup.py|   6 +-
 15 files changed, 213 insertions(+), 259 deletions(-)

diff --git a/bin/oonib b/bin/oonib
index 109c74e..ccbe246 100755
--- a/bin/oonib
+++ b/bin/oonib
@@ -2,55 +2,35 @@
 
 import sys
 import os
-from twisted.python import log, usage
-from twisted.internet import reactor
-from twisted.application import app
 
 # Hack to set the proper sys.path. Overcomes the export PYTHONPATH pain.
 sys.path[:] = map(os.path.abspath, sys.path)
 sys.path.insert(0, os.path.abspath(os.getcwd()))
 
-from oonib import errors as e
+from oonib import errors
 from oonib.config import config
-from oonib.log import LoggerFactory
-
 try:
 config.load()
-except e.ConfigFileNotSpecified:
+except errors.ConfigFileNotSpecified:
 print "Config file not specified!"
 print "Use -c to specify one!"
 config.usageOptions()
 sys.exit(1)
-except e.ConfigFileDoesNotExist, path:
+except errors.ConfigFileDoesNotExist, path:
 print "Config file \"%s\" does not exist!" % path
 sys.exit(2)
-except e.InvalidReportDirectory, path:
+except errors.InvalidReportDirectory, path:
 print "Invalid report directory: %s!" % path
 sys.exit(3)
-except e.InvalidArchiveDirectory, path:
+except errors.InvalidArchiveDirectory, path:
 print "Invalid archive directory: %s!" % path
 sys.exit(4)
-except e.InvalidInputDirectory, path:
+except errors.InvalidInputDirectory, path:
 print "Invalid input directory: %s" % path
 sys.exit(5)
-except e.InvalidDeckDirectory, path:
+except errors.InvalidDeckDirectory, path:
 print "Invalid deck directory: %s" % path
 sys.exit(6)
 
-if config.main.chroot:
-sys.argv.append('--chroot')
-sys.argv.append(config.chroot)
-
-if not config.main.nodaemon:
-sys.argv.append('-y')
-
-from oonib.oonibackend import application
-
-from twisted.scripts._twistd_unix import UnixApplicationRunner
-class OBaseRunner(UnixApplicationRunner):
-temporary_data_dir = None
-def createOrGetApplication(self):
-return application
-
-OBaseRunner.loggerFactory = LoggerFactory
-OBaseRunner(config.main).run()
+from oonib import oonibackend
+oonibackend.start()
diff --git a/data/archive/.gitignore b/data/archive/.gitignore
new file mode 100644
index 000..d6b7ef3
--- /dev/null
+++ b/data/archive/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
diff --git a/data/archive/empty.txt b/data/archive/empty.txt
deleted file mode 100644
index 9a166f9..000
--- a/data/archive/empty.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-In here will end up the archived reports.
-This file may be deleted at any time.
diff --git a/data/decks/.gitignore b/data/decks/.gitignore
new file mode 100644
index 000..514e47a
--- /dev/null
+++ b/data/decks/.gitignore
@@ -0,0 +1,3 @@
+*
+!.gitignore
+!README
diff --git a/data/inputs/.gitignore b/data/inputs/.gitignore
new file mode 100644
index 000..a0991ff
--- /dev/null
+++ b/data/inputs/.gitignore
@@ -0,0 +1,3 @@
+*
+!.gitignore
+!Makefile
diff --git a/data/reports/.gitignore b/data/reports/.gitignore
new file mode 100644
index 000..d6b7ef3
--- /dev/null
+++ b/data/reports/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
diff --git a/data/tor/.gitignore b/data/tor/.gitignore
new file mode 100644
index 000..d6b7ef3
--- /dev/null
+++ b/data/tor/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
diff --git a/data/tor/empty.txt b/data/tor/empty.txt
deleted file mode 100644
index ec3be8f..000
--- a/data/tor/empty.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-In here will end up all tor related files.
-This file may be deleted at any time.
diff --git a/oonib.conf.example b/oonib.conf.example
index 6f4c7f3..bcf8393 100644
--- a/oonib.conf.example
+++ b/oonib.conf.example
@@ -7,8 +7,6 @@ main:
 bouncer_file: data/bouncer.yaml
 
 logfile: null
-database_uri: 

[tor-commits] [oonib/master] Feature/web connectivity (#63)

[art]

commit 5178fc299d361c9ab57002e619e40b05061f5a7d
Author: Arturo Filastò 
Date:   Sat May 28 19:05:42 2016 +0200

Feature/web connectivity (#63)

* Add web connectivity test helper

* Forward test helper addresses to the host machine

* Align the request headers used by the web_connectivity test helper to 
those of the probe

* Add monkey patch for bug in twisted RedirectAgent:

https://twistedmatrix.com/trac/ticket/8265

* Add monkey patch for HTTPClientParser.statusReceived

* Use TrueHeaders in control request

* Add support for specifying endpoints in web_connectivity test helper

* Add endpoint for checking the status of the WebConnectivity test helper

* Add support for parsing test-helpers-alternate

* Fix key for web_connectivity test in example configuration file

* Implement on-disk web_connectivity cache

* Add support for Gzip content decoding

* Also record CNAME resolution.

* Rename ips to addrs

* Add support for retries in the http_request

* Add support for extracting title

* Encode the responses as well when debug mode

* Handle partial downloads

* Ignore errors when encoding headers

* Cast title to unicode and ignore errors

* Improvements based on feedback and comments by @bassosimone

* Move twisted related patches into txextra module

* Add support for returning the responses based on a key sent by the
  client.

* Inherit from OONIBHandler so we can get the error message format

* Stylistic improvements

* Set defaults in a way that oonib.conf can start from the example

* Avoid doing join on nonetype

* Address comments by @bassosimone

* Properly set the body also when we get a partial body downloaded

* Move more code into shared oonib.common module

* Fully sync the common module with ooni-probe (pulling in also other
  shared functionality so the Agents match entirely)

* Fix location of comment for the patched HTTPClient

* Add unittests and integration tests for web_connectivity
---
 Vagrantfile |  25 +++
 oonib.conf.example  |  18 +-
 oonib/bouncer/handlers.py   |  14 +-
 oonib/common/__init__.py|  10 ++
 oonib/common/http_utils.py  |  54 ++
 oonib/common/tcp_utils.py   |  10 ++
 oonib/common/txextra.py | 202 +
 oonib/oonibackend.py|  29 ++-
 oonib/test/test_web_connectivity.py |  71 
 oonib/testhelpers/http_helpers.py   | 344 +++-
 10 files changed, 758 insertions(+), 19 deletions(-)

diff --git a/Vagrantfile b/Vagrantfile
index b71e4e7..7aa5f71 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -4,6 +4,31 @@
 Vagrant.configure("2") do |config|
   config.vm.box = "precise32"
   config.vm.box_url = "http://files.vagrantup.com/precise32.box;
+
+  # Create a forwarded port mapping which allows access to a specific port
+  # within the machine from a port on the host machine. In the example below,
+  # accessing "localhost:8080" will access port 80 on the guest machine.
+  config.vm.network :forwarded_port, guest: 57001, host: 57001
+  config.vm.network :forwarded_port, guest: 57001, host: 57002
+  config.vm.network :forwarded_port, guest: 57001, host: 57003
+  config.vm.network :forwarded_port, guest: 57004, host: 57004
+  config.vm.network :forwarded_port, guest: 57005, host: 57005
+  config.vm.network :forwarded_port, guest: 57006, host: 57006
+  config.vm.network :forwarded_port, guest: 57007, host: 57007
+
+  # Create a private network, which allows host-only access to the machine
+  # using a specific IP.
+  # config.vm.network :private_network, ip: "192.168.33.10"
+
+  # Create a public network, which generally matched to bridged network.
+  # Bridged networks make the machine appear as another physical device on
+  # your network.
+  # config.vm.network :public_network
+
+  # Share an additional folder to the guest VM. The first argument is
+  # the path on the host to the actual folder. The second argument is
+  # the path on the guest to mount the folder. And the optional third
+  # argument is a set of non-required options.
   config.vm.synced_folder ".", "/data/oonib"
 end
 
diff --git a/oonib.conf.example b/oonib.conf.example
index 1925d81..6f4c7f3 100644
--- a/oonib.conf.example
+++ b/oonib.conf.example
@@ -30,12 +30,13 @@ main:
 tor_datadir: null
 
 bouncer_endpoints:
-- {type: tls, port: 10443, cert: "private/ssl-key-and-cert.pem"}
+#- {type: tls, port: 10443, cert: "private/ssl-key-and-cert.pem"}
 - {type: tcp, port: 10080}
-- {type: onion, hsdir: bouncer}
+- {type: onion, hsdir: /tmp/bouncer}
 
 collector_endpoints:
-- {type: tls, port: 11443, cert: 

[tor-commits] [oonib/master] pass tor_binary config to txtorcon (#79)

[art]

commit 0b6477c7af96663d2b0d853db66e413cdd599b30
Author: Joe Landers 
Date:   Mon May 23 17:51:52 2016 +0200

pass tor_binary config to txtorcon (#79)
---
 oonib/onion.py | 8 +---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/oonib/onion.py b/oonib/onion.py
index 8abefe5..0a7a155 100644
--- a/oonib/onion.py
+++ b/oonib/onion.py
@@ -106,13 +106,15 @@ def get_global_tor(reactor):
 
 try:
 if _global_tor_config is None:
-_global_tor_config = config = _configTor()
+_global_tor_config = _configTor()
 
 # start Tor launching
 def updates(prog, tag, summary):
 print("%d%%: %s" % (prog, summary))
-yield launch_tor(config, reactor, progress_updates=updates)
-yield config.post_bootstrap
+yield launch_tor(_global_tor_config, reactor,
+progress_updates=updates,
+tor_binary=config.main.tor_binary)
+yield _global_tor_config.post_bootstrap
 
 defer.returnValue(_global_tor_config)
 finally:



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [oonib/master] only assume HS endpoint when tor_hidden_service: true

[art]

commit 35ccea7dbf2b4b3a5d6d72c11f2bde34efd29286
Author: Joe Landers 
Date:   Fri May 13 09:33:24 2016 +0200

only assume HS endpoint when tor_hidden_service: true
---
 oonib/oonibackend.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py
index 3802566..eaf235f 100644
--- a/oonib/oonibackend.py
+++ b/oonib/oonibackend.py
@@ -157,10 +157,10 @@ if config.main.tor_hidden_service:
 configTor(torconfig)
 
 # this is to ensure same behaviour with an old config file
-if config.main.bouncer_endpoints is None:
+if config.main.bouncer_endpoints is None and config.main.tor_hidden_service:
 config.main.bouncer_endpoints = [ {'type': 'onion', 'hsdir': 'bouncer'} ]
 
-if config.main.collector_endpoints is None:
+if config.main.collector_endpoints is None and config.main.tor_hidden_service:
 config.main.collector_endpoints = [ {'type': 'onion', 'hsdir': 
'collector'} ]
 
 for endpoint_config in config.main.bouncer_endpoints:



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [oonib/master] Handle empty bouncer_endpoints and collector_endpoints

[art]

commit 9437610ec476c2c20150a905fbeac7e01908c8b7
Author: Arturo Filastò 
Date:   Sat May 21 21:19:52 2016 +0200

Handle empty bouncer_endpoints and collector_endpoints

* Set tor_data_dir to null in the default example
---
 oonib.conf.example   | 2 +-
 oonib/oonibackend.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/oonib.conf.example b/oonib.conf.example
index 0a5f4a5..95df448 100644
--- a/oonib.conf.example
+++ b/oonib.conf.example
@@ -28,7 +28,7 @@ main:
 stale_time: 3600
 
 tor_hidden_service: false
-tor_datadir: test_datadir
+tor_datadir: null
 
 bouncer_endpoints:
 - {type: tls, port: 10443, cert: "private/ssl-key-and-cert.pem"}
diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py
index 18bacf0..33ea92d 100644
--- a/oonib/oonibackend.py
+++ b/oonib/oonibackend.py
@@ -162,12 +162,12 @@ if config.main.tor_hidden_service and \
 config.main.bouncer_endpoints   = [ {'type': 'onion', 'hsdir':   
bouncer_hsdir} ]
 config.main.collector_endpoints = [ {'type': 'onion', 'hsdir': 
collector_hsdir} ]
 
-for endpoint_config in config.main.get('bouncer_endpoints'):
+for endpoint_config in config.main.get('bouncer_endpoints', []):
 print "Starting bouncer with config %s" % endpoint_config
 endpoint = getEndpoint(endpoint_config)
 createService(endpoint, 'bouncer', endpoint_config)
 
-for endpoint_config in config.main.get('collector_endpoints'):
+for endpoint_config in config.main.get('collector_endpoints', []):
 print "Starting collector with config %s" % endpoint_config
 endpoint = getEndpoint(endpoint_config)
 createService(endpoint, 'collector', endpoint_config)



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [oonib/master] Ensure bouncer_endpoints and collector_endpoints are by default empty lists

[art]

commit 90ca955e970ef30c5963acce77c5f736372fc440
Author: Arturo Filastò 
Date:   Fri May 13 13:20:28 2016 +0200

Ensure bouncer_endpoints and collector_endpoints are by default empty lists
---
 oonib/oonibackend.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py
index 5a994e8..337b797 100644
--- a/oonib/oonibackend.py
+++ b/oonib/oonibackend.py
@@ -163,6 +163,9 @@ if config.main.bouncer_endpoints is None and 
config.main.tor_hidden_service:
 if config.main.collector_endpoints is None and config.main.tor_hidden_service:
 config.main.collector_endpoints = [ {'type': 'onion', 'hsdir': 
'collector'} ]
 
+config.main.bouncer_endpoints = config.main.get('bouncer_endpoints', [])
+config.main.collector_endpoints = config.main.get('collector_endpoints', [])
+
 for endpoint_config in config.main.bouncer_endpoints:
 print "Starting bouncer with config %s" % endpoint_config
 endpoint = getEndpoint(endpoint_config)
@@ -172,4 +175,3 @@ for endpoint_config in config.main.collector_endpoints:
 print "Starting collector with config %s" % endpoint_config
 endpoint = getEndpoint(endpoint_config)
 createService(endpoint, 'collector', endpoint_config)
-



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [oonib/master] [hotfix] Only create bouncer endpoints when it's configured

[art]

commit f03f8055d44d3bbb7cbcfb789946ba67a94e1e52
Author: Arturo Filastò 
Date:   Thu May 26 13:49:28 2016 +0200

[hotfix] Only create bouncer endpoints when it's configured
---
 oonib/oonibackend.py | 9 ++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py
index 33ea92d..601a972 100644
--- a/oonib/oonibackend.py
+++ b/oonib/oonibackend.py
@@ -163,9 +163,12 @@ if config.main.tor_hidden_service and \
 config.main.collector_endpoints = [ {'type': 'onion', 'hsdir': 
collector_hsdir} ]
 
 for endpoint_config in config.main.get('bouncer_endpoints', []):
-print "Starting bouncer with config %s" % endpoint_config
-endpoint = getEndpoint(endpoint_config)
-createService(endpoint, 'bouncer', endpoint_config)
+if config.main.bouncer_file:
+print "Starting bouncer with config %s" % endpoint_config
+endpoint = getEndpoint(endpoint_config)
+createService(endpoint, 'bouncer', endpoint_config)
+else:
+print "No bouncer configured"
 
 for endpoint_config in config.main.get('collector_endpoints', []):
 print "Starting collector with config %s" % endpoint_config



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [oonib/master] Map also the connect_error (#85)

[art]

commit 2b92f6327fc2bcb4da06e686fbc3b5b4fc4bbbe6
Author: Arturo Filastò 
Date:   Mon May 30 15:39:05 2016 +0200

Map also the connect_error (#85)
---
 oonib/testhelpers/http_helpers.py | 9 -
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/oonib/testhelpers/http_helpers.py 
b/oonib/testhelpers/http_helpers.py
index 1cddf24..a771a19 100644
--- a/oonib/testhelpers/http_helpers.py
+++ b/oonib/testhelpers/http_helpers.py
@@ -11,8 +11,10 @@ from cyclone.web import RequestHandler, Application, 
HTTPError
 from cyclone.web import asynchronous
 from twisted.internet import protocol, defer, reactor
 from twisted.internet.endpoints import TCP4ClientEndpoint
-from twisted.internet.error import ConnectionRefusedError
+
+from twisted.internet.error import ConnectionRefusedError, ConnectError
 from twisted.internet.error import DNSLookupError, TimeoutError
+
 from twisted.names import client as dns_client
 from twisted.names import dns
 from twisted.names.error import DNSNameError, DNSServerError
@@ -359,6 +361,8 @@ class WebConnectivityCache(object):
 page_info['failure'] = 'generic_timeout_error'
 except ConnectionRefusedError:
 page_info['failure'] = 'connection_refused_error'
+except ConnectError:
+page_info['failure'] = 'connect_error'
 except:
 # XXX map more failures
 page_info['failure'] = 'unknown_error'
@@ -390,6 +394,9 @@ class WebConnectivityCache(object):
 except ConnectionRefusedError:
 socket_info['status'] = False
 socket_info['failure'] = 'connection_refused_error'
+except ConnectError:
+socket_info['status'] = False
+socket_info['failure'] = 'connect_error'
 except:
 socket_info['status'] = False
 socket_info['failure'] = 'unknown_error'

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [oonib/master] Vagrant: use box canonical name rather than URL (#84)

[art]

commit 973c9b0c2ae089e3043253890148d25dd550f756
Author: Simone Basso 
Date:   Mon May 30 10:50:21 2016 +0200

Vagrant: use box canonical name rather than URL (#84)

This allows us to get updates running `vagrant box update`.

While at it, add `/.vagrant/` to `.gitignore`.
---
 .gitignore  | 3 +++
 Vagrantfile | 3 +--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/.gitignore b/.gitignore
index 7fc5cbd..3f4029e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -63,3 +63,6 @@ installed-files.txt
 oonib.conf
 
 oonib.pid
+
+# Local vagrant files
+/.vagrant/
diff --git a/Vagrantfile b/Vagrantfile
index 7aa5f71..a2abe0d 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -2,8 +2,7 @@
 # vi: set ft=ruby :
 
 Vagrant.configure("2") do |config|
-  config.vm.box = "precise32"
-  config.vm.box_url = "http://files.vagrantup.com/precise32.box;
+  config.vm.box = "ubuntu/precise32"
 
   # Create a forwarded port mapping which allows access to a specific port
   # within the machine from a port on the host machine. In the example below,



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [oonib/master] keep old config behavior with new endpoint code

[art]

commit 575f0aa5a44d1f851fb3d4fbb1bcf1fbf36a4f2d
Author: Joe Landers 
Date:   Sun May 15 18:25:41 2016 +0200

keep old config behavior with new endpoint code
---
 oonib/onion.py   | 36 
 oonib/oonibackend.py | 42 +++---
 2 files changed, 51 insertions(+), 27 deletions(-)

diff --git a/oonib/onion.py b/oonib/onion.py
index 262e672..8abefe5 100644
--- a/oonib/onion.py
+++ b/oonib/onion.py
@@ -1,7 +1,7 @@
 import tempfile
 from oonib import log
 from oonib.config import config
-from twisted.internet import reactor, endpoints
+from twisted.internet import reactor, endpoints, defer
 import os
 
 from random import randint
@@ -53,9 +53,8 @@ def txSetupFailed(failure):
 log.err("Setup failed")
 log.exception(failure)
 
-def configTor(torconfig):
-def updates(prog, tag, summary):
-print("%d%%: %s" % (prog, summary))
+def _configTor():
+torconfig = TorConfig()
 
 if config.main.socks_port:
 torconfig.SocksPort = config.main.socks_port
@@ -89,3 +88,32 @@ def configTor(torconfig):
 config.main.socks_port = socks_port
 
 torconfig.save()
+return torconfig
+
+# get_global_tor is a near-rip of that from txtorcon (so you can have some
+# confidence in the logic of it), but we use our own _configTor() while
+# the txtorcon function hardcodes some default values we don't want.
+_global_tor_config = None
+_global_tor_lock = defer.DeferredLock()
+# we need the lock because we (potentially) yield several times while
+# "creating" the TorConfig instance
+
+@defer.inlineCallbacks
+def get_global_tor(reactor):
+global _global_tor_config
+global _global_tor_lock
+yield _global_tor_lock.acquire()
+
+try:
+if _global_tor_config is None:
+_global_tor_config = config = _configTor()
+
+# start Tor launching
+def updates(prog, tag, summary):
+print("%d%%: %s" % (prog, summary))
+yield launch_tor(config, reactor, progress_updates=updates)
+yield config.post_bootstrap
+
+defer.returnValue(_global_tor_config)
+finally:
+_global_tor_lock.release()
diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py
index 337b797..18bacf0 100644
--- a/oonib/oonibackend.py
+++ b/oonib/oonibackend.py
@@ -11,14 +11,14 @@ from distutils.version import LooseVersion
 
 from oonib.api import ooniBackend, ooniBouncer
 from oonib.config import config
-from oonib.onion import configTor
+from oonib.onion import get_global_tor
 from oonib.testhelpers import dns_helpers, ssl_helpers
 from oonib.testhelpers import http_helpers, tcp_helpers
 
 import os
 
 from twisted.application import internet, service
-from twisted.internet import reactor, endpoints, ssl
+from twisted.internet import reactor, endpoints, ssl, defer
 from twisted.names import dns
 
 from txtorcon import TCPHiddenServiceEndpoint, TorConfig
@@ -103,15 +103,17 @@ if config.helpers['http-return-json-headers'].port:
 http_return_request_helper.startService()
 
 def getHSEndpoint(endpoint_config):
-if torconfig is None:
-raise Exception("you probably need to set tor_hidden_service: true")
-hsdir = os.path.join(torconfig.DataDirectory, endpoint_config['hsdir'])
+hsdir = endpoint_config['hsdir']
+hsdir = os.path.expanduser(hsdir)
+hsdir = os.path.realpath(hsdir)
 if LooseVersion(txtorcon_version) >= LooseVersion('0.10.0'):
-return TCPHiddenServiceEndpoint.global_tor(reactor,
+return TCPHiddenServiceEndpoint(reactor,
+get_global_tor(reactor),
 80,
 hidden_service_dir=hsdir)
 else:
-return TCPHiddenServiceEndpoint.global_tor(reactor,
+return TCPHiddenServiceEndpoint(reactor,
+get_global_tor(reactor),
 80,
 data_dir=hsdir)
 
@@ -151,27 +153,21 @@ def createService(endpoint, role, endpoint_config):
 multiService.addService(service)
 service.startService()
 
-torconfig = None
-if config.main.tor_hidden_service:
-torconfig = TorConfig()
-configTor(torconfig)
-
 # this is to ensure same behaviour with an old config file
-if config.main.bouncer_endpoints is None and config.main.tor_hidden_service:
-config.main.bouncer_endpoints = [ {'type': 'onion', 'hsdir': 'bouncer'} ]
-
-if config.main.collector_endpoints is None and config.main.tor_hidden_service:
-config.main.collector_endpoints = [ {'type': 'onion', 'hsdir': 
'collector'} ]
-
-config.main.bouncer_endpoints = config.main.get('bouncer_endpoints', [])
-config.main.collector_endpoints = config.main.get('collector_endpoints', [])
-
-for endpoint_config in config.main.bouncer_endpoints:
+if config.main.tor_hidden_service and \
+config.main.bouncer_endpoints 

[tor-commits] [oonib/master] Fix typo in gen-ssl-key-cert script

[art]

commit d7f9a4c731acd9979376fa73438d4e8afae75446
Author: Arturo Filastò 
Date:   Fri May 13 12:57:38 2016 +0200

Fix typo in gen-ssl-key-cert script
---
 scripts/gen-ssl-key-cert.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/gen-ssl-key-cert.sh b/scripts/gen-ssl-key-cert.sh
index 2a50766..da85b58 100755
--- a/scripts/gen-ssl-key-cert.sh
+++ b/scripts/gen-ssl-key-cert.sh
@@ -10,4 +10,4 @@ openssl req -x509 -newkey rsa:2048 \
 -keyout private/ssl-key.pem -out private/ssl-cert.pem \
 -days 400 -nodes -subj '/CN=selfie'
 
-cat private/ssl-key.pem private/ssh-cert.pem > private/ssl-key-and-cert.pem
+cat private/ssl-key.pem private/ssl-cert.pem > private/ssl-key-and-cert.pem



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [oonib/master] Remove unused imports

[art]

commit 0bd12a10549d214fff93c4b11e14b5d62bafc0ea
Author: Arturo Filastò 
Date:   Fri May 13 13:18:05 2016 +0200

Remove unused imports
---
 oonib/oonibackend.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py
index eaf235f..5a994e8 100644
--- a/oonib/oonibackend.py
+++ b/oonib/oonibackend.py
@@ -18,7 +18,7 @@ from oonib.testhelpers import http_helpers, tcp_helpers
 import os
 
 from twisted.application import internet, service
-from twisted.internet import reactor, endpoints, defer, ssl, protocol
+from twisted.internet import reactor, endpoints, ssl
 from twisted.names import dns
 
 from txtorcon import TCPHiddenServiceEndpoint, TorConfig



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [oonib/master] Merge branch 'feature/https_endpoint'

[art]

commit 612eb5a1c7a6ea1f2f83d3eb7a9f5ffb1d68c64f
Merge: 94d3452 90ca955
Author: Arturo Filastò 
Date:   Fri May 13 13:21:28 2016 +0200

Merge branch 'feature/https_endpoint'

* feature/https_endpoint:
  Ensure bouncer_endpoints and collector_endpoints are by default empty 
lists
  Remove unused imports
  Fix typo in gen-ssl-key-cert script
  only assume HS endpoint when tor_hidden_service: true
  explain + bail out when inconsistent config
  Move gen-ssl-key-cert to scripts directory
  remove another print from debugging
  maintain old behaviour
  remove echo server i used for debugging
  easily swappable {TCP, TLS, onion} endpoints

 oonib.conf.example  |  13 -
 oonib/onion.py  |  10 +---
 oonib/oonibackend.py| 130 +---
 scripts/gen-ssl-key-cert.sh |  13 +
 4 files changed, 98 insertions(+), 68 deletions(-)



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [oonib/master] Download openssl with wget as well to fix SSL errors on MLAB

[art]

commit 94d34521e9ec7f6a61b1d486dbc53506cb9034cb
Author: Arturo Filastò 
Date:   Tue May 10 19:06:44 2016 +0200

Download openssl with wget as well to fix SSL errors on MLAB
---
 scripts/build_tor2web_tor.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/build_tor2web_tor.sh b/scripts/build_tor2web_tor.sh
index 080a878..c9d7d1b 100755
--- a/scripts/build_tor2web_tor.sh
+++ b/scripts/build_tor2web_tor.sh
@@ -12,12 +12,12 @@ SCRIPT_ROOT=`pwd`
 # Package URLS
 CURL_URLS="\
 http://zlib.net/zlib-$ZLIB_VERSION.tar.gz
-https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz.asc
-https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz
 
https://github.com/downloads/libevent/libevent/libevent-$LIBEVENT_VERSION.tar.gz.asc
 
https://github.com/downloads/libevent/libevent/libevent-$LIBEVENT_VERSION.tar.gz;
 
 WGET_URLS="\
+https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz.asc
+https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz
 https://dist.torproject.org/tor-$TOR_VERSION.tar.gz
 https://dist.torproject.org/tor-$TOR_VERSION.tar.gz.asc;
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [oonib/master] maintain old behaviour

[art]

commit 5fa24c3364c4349995d3bb1ca900dc54e546f5c4
Author: Joe Landers 
Date:   Fri May 6 13:11:33 2016 +0200

maintain old behaviour
---
 oonib/oonibackend.py | 27 ---
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py
index 0b8c4ec..1e77d6b 100644
--- a/oonib/oonibackend.py
+++ b/oonib/oonibackend.py
@@ -154,15 +154,20 @@ if config.main.tor_hidden_service:
 torconfig = TorConfig()
 configTor(torconfig)
 
-if config.main.bouncer_endpoints:
-for endpoint_config in config.main.bouncer_endpoints:
-print "Starting bouncer with config %s" % endpoint_config
-endpoint = getEndpoint(endpoint_config)
-createService(endpoint, 'bouncer', endpoint_config)
-
-if config.main.bouncer_endpoints:
-for endpoint_config in config.main.collector_endpoints:
-print "Starting collector with config %s" % endpoint_config
-endpoint = getEndpoint(endpoint_config)
-createService(endpoint, 'collector', endpoint_config)
+# this is to ensure same behaviour with an old config file
+if config.main.bouncer_endpoints is None:
+config.main.bouncer_endpoints = [ {'type': 'onion', 'hsdir': 'bouncer'} ]
+
+if config.main.collector_endpoints is None:
+config.main.collector_endpoints = [ {'type': 'onion', 'hsdir': 
'collector'} ]
+
+for endpoint_config in config.main.bouncer_endpoints:
+print "Starting bouncer with config %s" % endpoint_config
+endpoint = getEndpoint(endpoint_config)
+createService(endpoint, 'bouncer', endpoint_config)
+
+for endpoint_config in config.main.collector_endpoints:
+print "Starting collector with config %s" % endpoint_config
+endpoint = getEndpoint(endpoint_config)
+createService(endpoint, 'collector', endpoint_config)
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits