[tor-commits] [torbirdy/master] Replace 'var' with 'let' for iterations
commit de8d9fa2964132f657f03d3ba27f3ea317c39eba Author: Sukhbir SinghDate: Mon May 30 06:19:48 2016 -0400 Replace 'var' with 'let' for iterations --- chrome/content/preferences.js | 10 +- components/torbirdy.js| 26 +- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/chrome/content/preferences.js b/chrome/content/preferences.js index 682b367..c843d3c 100644 --- a/chrome/content/preferences.js +++ b/chrome/content/preferences.js @@ -77,7 +77,7 @@ if (!org.torbirdy.prefs) org.torbirdy.prefs = new function() { pub.clearCustomPrefs = function() { var customPrefs = pub.torBirdyPref.getChildList("", {}); -for (var i = 0; i < customPrefs.length; i++) { +for (let i = 0; i < customPrefs.length; i++) { pub.prefs.clearUserPref(pub.customBranch + customPrefs[i]); } }; @@ -131,7 +131,7 @@ if (!org.torbirdy.prefs) org.torbirdy.prefs = new function() { var accountLength = accounts.length; -for (var i = 0; i < accountLength; i++) { +for (let i = 0; i < accountLength; i++) { var account = accounts.queryElementAt(i, Components.interfaces.nsIMsgAccount).incomingServer; var name = account.prettyName; @@ -149,7 +149,7 @@ if (!org.torbirdy.prefs) org.torbirdy.prefs = new function() { // For email accounts, configure accordingly. var sAccount = null; var account = pub.getAccount(); - for (var i = 0; i < account.length; i++) { + for (let i = 0; i < account.length; i++) { if (account[i].key === pub.mailAccount.value) { sAccount = i; } @@ -324,14 +324,14 @@ if (!org.torbirdy.prefs) org.torbirdy.prefs = new function() { var accounts = pub.getAccount(); if (pub.fetchAllMails.checked) { pub.prefs.setBoolPref(pub.prefBranch + 'fetchall', true); -for (var i = 0; i < accounts.length; i++) { +for (let i = 0; i < accounts.length; i++) { accounts[i].loginAtStartUp = true; accounts[i].doBiff = true; } } else { pub.prefs.setBoolPref(pub.prefBranch + 'fetchall', false); -for (var i = 0; i < accounts.length; i++) { +for (let i = 0; i < accounts.length; i++) { accounts[i].loginAtStartUp = false; accounts[i].doBiff = false; } diff --git a/components/torbirdy.js b/components/torbirdy.js index 9c930e8..fa1aa6b 100644 --- a/components/torbirdy.js +++ b/components/torbirdy.js @@ -480,13 +480,13 @@ TorBirdy.prototype = { resetUserPrefs: function() { dump("Resetting user preferences to default\n"); // Clear the Thunderbird preferences we changed. -for (var each in TorBirdyPrefs) { +for (let each in TorBirdyPrefs) { this.prefs.clearUserPref(each); } // Restore the older proxy preferences that were set prior to TorBirdy's install. dump("Restoring proxy settings\n"); -for (var i = 0; i < TorBirdyOldPrefs.length; i++) { +for (let i = 0; i < TorBirdyOldPrefs.length; i++) { var oldPref = TorBirdyOldPrefs[i]; var setValue = kRestoreBranch + oldPref; var type = this.prefs.getPrefType(setValue); @@ -502,14 +502,14 @@ TorBirdy.prototype = { } // Enable plugins. -for(var i = 0; i < this.plugins.length; i++) { +for(let i = 0; i < this.plugins.length; i++) { this.plugins[i].disabled = false; } // Now clear all TorBirdy preferences. var clearPrefs = Cc["@mozilla.org/preferences-service;1"] .getService(Ci.nsIPrefService).getBranch(kTorBirdyBranch).getChildList("", {}); -for (var i = 0; i < clearPrefs.length; i++) { +for (let i = 0; i < clearPrefs.length; i++) { this.prefs.clearUserPref(kTorBirdyBranch + clearPrefs[i]); } }, @@ -517,7 +517,7 @@ TorBirdy.prototype = { setPrefs: function() { // If custom values are set for specific preferences, override the defaults with them. // For each preference, get the type and then set the property. -for (var i = 0; i < this.customPrefs.length; i++) { +for (let i = 0; i < this.customPrefs.length; i++) { var typePref = this.prefs.getPrefType(this.customPrefs[i]); // String. if (typePref === 32) { @@ -534,7 +534,7 @@ TorBirdy.prototype = { TorBirdyPrefs[this.customPrefs[i]] = value; } -for (var each in TorBirdyPrefs) { +for (let each in TorBirdyPrefs) { if (typeof TorBirdyPrefs[each] === "boolean") { this.prefs.setBoolPref(each, TorBirdyPrefs[each]); } @@ -554,7 +554,7 @@ TorBirdy.prototype = { // First copy TorBirdyPrefs to TorBirdyOldPrefs. TorBirdyOldPrefs.push.apply(TorBirdyOldPrefs, Object.keys(TorBirdyPrefs)); - for (var i = 0; i < TorBirdyOldPrefs.length; i++) { + for (let i = 0; i < TorBirdyOldPrefs.length; i++) { var oldPref = TorBirdyOldPrefs[i]; var type =
[tor-commits] [tor/master] Let's not even talk about those errors, ok?
commit 3cdc8bfa2c995b3b381e462128c0408e0ae5b3af Author: Nick MathewsonDate: Mon May 30 17:14:46 2016 -0400 Let's not even talk about those errors, ok? --- src/test/test_dir_handle_get.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index e82eddf..7f92c8c 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -1013,7 +1013,7 @@ test_dir_handle_get_server_descriptors_d(void* data) conn = dir_connection_new(tor_addr_family(_TOR_ADDR)); char req_header[155]; /* XXX Why 155? What kind of number is that?? */ - tor_snprintf(req_header, sizeof(req_handler), + tor_snprintf(req_header, sizeof(req_header), SERVER_DESC_GET("d/%s+" HEX1 "+" HEX2), hex_digest); tt_int_op(directory_handle_command_get(conn, req_header, NULL, 0), OP_EQ, 0); @@ -1358,7 +1358,7 @@ test_dir_handle_get_server_keys_fp(void* data) conn = dir_connection_new(tor_addr_family(_TOR_ADDR)); char req[71]; - tor_snprintf(req, (sizeof(req), + tor_snprintf(req, sizeof(req), GET("/tor/keys/fp/%s"), TEST_CERT_IDENT_KEY); tt_int_op(directory_handle_command_get(conn, req, NULL, 0), OP_EQ, 0); ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [tor/master] Wait, we had sprintf() in our unit tests?? FOR SHAME!
commit 97f2c1c58e0ad0a60b3c0ce35921e444c93b6e4f Author: Nick MathewsonDate: Mon May 30 16:50:57 2016 -0400 Wait, we had sprintf() in our unit tests?? FOR SHAME! --- src/test/test_dir_handle_get.c | 34 -- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index 3029e98..e82eddf 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -393,7 +393,7 @@ test_dir_handle_get_rendezvous2_on_encrypted_conn_success(void *data) TO_CONN(conn)->linked = 1; tt_assert(connection_dir_is_encrypted(conn)); - sprintf(req, RENDEZVOUS2_GET("%s"), desc_id_base32); + tor_snprintf(req, sizeof(req), RENDEZVOUS2_GET("%s"), desc_id_base32); tt_int_op(directory_handle_command_get(conn, req, NULL, 0), OP_EQ, 0); @@ -408,7 +408,7 @@ test_dir_handle_get_rendezvous2_on_encrypted_conn_success(void *data) tt_assert(strstr(header, "Content-Type: text/plain\r\n")); tt_assert(strstr(header, "Content-Encoding: identity\r\n")); tt_assert(strstr(header, "Pragma: no-cache\r\n")); - sprintf(buff, "Content-Length: %ld\r\n", (long) body_len); + tor_snprintf(buff, sizeof(buff), "Content-Length: %ld\r\n", (long) body_len); tt_assert(strstr(header, buff)); tt_int_op(body_used, OP_EQ, strlen(body)); @@ -520,7 +520,7 @@ test_dir_handle_get_micro_d(void *data) /* Make the request */ conn = dir_connection_new(tor_addr_family(_TOR_ADDR)); - sprintf(path, MICRODESC_GET("%s"), digest_base64); + tor_snprintf(path, sizeof(path), MICRODESC_GET("%s"), digest_base64); tt_int_op(directory_handle_command_get(conn, path, NULL, 0), OP_EQ, 0); fetch_from_buf_http(TO_CONN(conn)->outbuf, , MAX_HEADERS_SIZE, @@ -590,7 +590,7 @@ test_dir_handle_get_micro_d_server_busy(void *data) /* Make the request */ conn = dir_connection_new(tor_addr_family(_TOR_ADDR)); - sprintf(path, MICRODESC_GET("%s"), digest_base64); + tor_snprintf(path, sizeof(path), MICRODESC_GET("%s"), digest_base64); tt_int_op(directory_handle_command_get(conn, path, NULL, 0), OP_EQ, 0); fetch_from_buf_http(TO_CONN(conn)->outbuf, , MAX_HEADERS_SIZE, @@ -952,7 +952,8 @@ test_dir_handle_get_server_descriptors_fp(void* data) DIGEST_LEN); char req[155]; - sprintf(req, SERVER_DESC_GET("fp/%s+" HEX1 "+" HEX2), hex_digest); + tor_snprintf(req, sizeof(req), SERVER_DESC_GET("fp/%s+" HEX1 "+" HEX2), + hex_digest); tt_int_op(directory_handle_command_get(conn, req, NULL, 0), OP_EQ, 0); //TODO: Is this a BUG? @@ -1011,8 +1012,9 @@ test_dir_handle_get_server_descriptors_d(void* data) conn = dir_connection_new(tor_addr_family(_TOR_ADDR)); - char req_header[155]; - sprintf(req_header, SERVER_DESC_GET("d/%s+" HEX1 "+" HEX2), hex_digest); + char req_header[155]; /* XXX Why 155? What kind of number is that?? */ + tor_snprintf(req_header, sizeof(req_handler), + SERVER_DESC_GET("d/%s+" HEX1 "+" HEX2), hex_digest); tt_int_op(directory_handle_command_get(conn, req_header, NULL, 0), OP_EQ, 0); //TODO: Is this a BUG? @@ -1080,8 +1082,9 @@ test_dir_handle_get_server_descriptors_busy(void* data) #define HEX1 "Fe0daff89127389bc67558691231234551193EEE" #define HEX2 "Deadbeef999119f00ba4" - char req_header[155]; - sprintf(req_header, SERVER_DESC_GET("d/%s+" HEX1 "+" HEX2), hex_digest); + char req_header[155]; /* XXX 155? Why 155? */ + tor_snprintf(req_header, sizeof(req_header), + SERVER_DESC_GET("d/%s+" HEX1 "+" HEX2), hex_digest); tt_int_op(directory_handle_command_get(conn, req_header, NULL, 0), OP_EQ, 0); fetch_from_buf_http(TO_CONN(conn)->outbuf, , MAX_HEADERS_SIZE, @@ -1355,7 +1358,8 @@ test_dir_handle_get_server_keys_fp(void* data) conn = dir_connection_new(tor_addr_family(_TOR_ADDR)); char req[71]; - sprintf(req, GET("/tor/keys/fp/%s"), TEST_CERT_IDENT_KEY); + tor_snprintf(req, (sizeof(req), + GET("/tor/keys/fp/%s"), TEST_CERT_IDENT_KEY); tt_int_op(directory_handle_command_get(conn, req, NULL, 0), OP_EQ, 0); fetch_from_buf_http(TO_CONN(conn)->outbuf, , MAX_HEADERS_SIZE, @@ -1427,7 +1431,8 @@ test_dir_handle_get_server_keys_sk(void* data) conn = dir_connection_new(tor_addr_family(_TOR_ADDR)); char req[71]; - sprintf(req, GET("/tor/keys/sk/%s"), TEST_SIGNING_KEY); + tor_snprintf(req, sizeof(req), + GET("/tor/keys/sk/%s"), TEST_SIGNING_KEY); tt_int_op(directory_handle_command_get(conn, req, NULL, 0), OP_EQ, 0); fetch_from_buf_http(TO_CONN(conn)->outbuf, , MAX_HEADERS_SIZE, @@ -1510,8 +1515,9 @@ test_dir_handle_get_server_keys_fpsk(void* data) conn = dir_connection_new(tor_addr_family(_TOR_ADDR)); char req[115]; - sprintf(req, GET("/tor/keys/fp-sk/%s-%s"), - TEST_CERT_IDENT_KEY, TEST_SIGNING_KEY); + tor_snprintf(req, sizeof(req), +
[tor-commits] [tor/master] Replace nearly all XXX0vv comments with smarter ones
commit 4f1a04ff9c74456c03ac67adb7ae780e4e77c71e Author: Nick MathewsonDate: Mon May 30 16:18:16 2016 -0400 Replace nearly all XXX0vv comments with smarter ones So, back long ago, XXX012 meant, "before Tor 0.1.2 is released, we had better revisit this comment and fix it!" But we have a huge pile of such comments accumulated for a large number of released versions! Not cool. So, here's what I tried to do: * 0.2.9 and 0.2.8 are retained, since those are not yet released. * XXX+ or XXX++ or XXX or whatever means, "This one looks quite important!" * The others, after one-by-one examination, are downgraded to plain old XXX. Which doesn't mean they aren't a problem -- just that they cannot possibly be a release-blocking problem. --- src/common/address.c | 2 +- src/common/compat.c | 3 ++- src/common/procmon.c | 9 + src/ext/eventdns.c | 3 +-- src/or/buffers.c | 6 +++--- src/or/circpathbias.c| 5 ++--- src/or/circuitbuild.c| 9 - src/or/circuituse.c | 12 ++-- src/or/config.c | 12 +++- src/or/config.h | 2 +- src/or/connection.c | 10 +- src/or/connection_edge.c | 10 +- src/or/connection_or.c | 2 +- src/or/control.c | 4 ++-- src/or/directory.c | 9 + src/or/dirserv.c | 6 +++--- src/or/dirvote.c | 2 +- src/or/entrynodes.c | 7 --- src/or/main.c| 2 +- src/or/microdesc.c | 2 +- src/or/networkstatus.c | 2 +- src/or/onion.c | 2 +- src/or/or.h | 10 +- src/or/rendclient.c | 6 +++--- src/or/rephist.c | 2 +- src/or/router.c | 2 +- src/or/routerlist.c | 6 +++--- src/or/routerparse.c | 4 ++-- src/or/transports.c | 2 +- src/test/test_dir.c | 2 +- 30 files changed, 75 insertions(+), 80 deletions(-) diff --git a/src/common/address.c b/src/common/address.c index a6e0f3f..01607a4 100644 --- a/src/common/address.c +++ b/src/common/address.c @@ -1809,7 +1809,7 @@ MOCK_IMPL(smartlist_t *,get_interface_address6_list,(int severity, /* == * IPv4 helpers - * 024 IPv6 deprecate some of these. + * IPv6 deprecate some of these. */ /** Given an address of the form "ip:port", try to divide it into its diff --git a/src/common/compat.c b/src/common/compat.c index 23eaa13..9d49b8f 100644 --- a/src/common/compat.c +++ b/src/common/compat.c @@ -15,7 +15,8 @@ /* This is required on rh7 to make strptime not complain. * We also need it to make memmem get defined (where available) */ -/* 024 We should just use AC_USE_SYSTEM_EXTENSIONS in our autoconf, + +/* We should just use AC_USE_SYSTEM_EXTENSIONS in our autoconf, * and get this (and other important stuff!) automatically. Once we do that, * make sure to also change the extern char **environ detection in * configure.ac, because whether that is declared or not depends on whether diff --git a/src/common/procmon.c b/src/common/procmon.c index 12d53fc..4ecee26 100644 --- a/src/common/procmon.c +++ b/src/common/procmon.c @@ -116,11 +116,11 @@ struct tor_process_monitor_t { * periodically check whether the process we have a handle to has * ended. */ HANDLE hproc; - /* XXX023 We can and should have Libevent watch hproc for us, - * if/when some version of Libevent 2.x can be told to do so. */ + /* We should have Libevent watch hproc for us, + * if/when some version of Libevent can be told to do so. */ #endif - /* XXX023 On Linux, we can and should receive the 22nd + /* On Linux, we can and should receive the 22nd * (space-delimited) field (âstarttimeâ) of /proc/$PID/stat from the * owning controller and store it, and poll once in a while to see * whether it has changed -- if so, the kernel has *definitely* @@ -130,7 +130,8 @@ struct tor_process_monitor_t { * systems whose admins have mounted procfs, or the start-time field * of the process-information structure returned by kvmgetprocs() on * any system. The latter is ickier. */ - /* XXX023 On FreeBSD (and possibly other kqueue systems), we can and + + /* On FreeBSD (and possibly other kqueue systems), we can and * should arrange to receive EVFILT_PROC NOTE_EXIT notifications for * pid, so we don't have to do such a heavyweight poll operation in * order to avoid the PID-reassignment race condition. (We would diff --git a/src/ext/eventdns.c b/src/ext/eventdns.c index fc5657c..7046f32 100644 --- a/src/ext/eventdns.c +++ b/src/ext/eventdns.c @@ -2004,8 +2004,7 @@ evdns_request_timeout_callback(int fd, short events, void *arg) { } else { /* retransmit it */ /* Stop waiting for the timeout. No need to do this in -* request_finished; that one already deletes
[tor-commits] [tor/master] We no longer generate v0 directories. Remove the code to do so
commit ce31db4326599c3f86307fca5286349150cb871e Author: Nick MathewsonDate: Mon May 30 16:05:37 2016 -0400 We no longer generate v0 directories. Remove the code to do so --- src/or/dirserv.c | 88 src/or/dirserv.h | 2 -- 2 files changed, 90 deletions(-) diff --git a/src/or/dirserv.c b/src/or/dirserv.c index d1ea5df..e61f56e 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -985,94 +985,6 @@ router_is_active(const routerinfo_t *ri, const node_t *node, time_t now) return 1; } -/** Generate a new v1 directory and write it into a newly allocated string. - * Point *dir_out to the allocated string. Sign the - * directory with private_key. Return 0 on success, -1 on - * failure. If complete is set, give us all the descriptors; - * otherwise leave out non-running and non-valid ones. - */ -int -dirserv_dump_directory_to_string(char **dir_out, - crypto_pk_t *private_key) -{ - /* 024 Get rid of this function if we can confirm that nobody's - * fetching these any longer */ - char *cp; - char *identity_pkey; /* Identity key, DER64-encoded. */ - char *recommended_versions; - char digest[DIGEST_LEN]; - char published[ISO_TIME_LEN+1]; - char *buf = NULL; - size_t buf_len; - size_t identity_pkey_len; - time_t now = time(NULL); - - tor_assert(dir_out); - *dir_out = NULL; - - if (crypto_pk_write_public_key_to_string(private_key,_pkey, - _pkey_len)<0) { -log_warn(LD_BUG,"write identity_pkey to string failed!"); -return -1; - } - - recommended_versions = -format_versions_list(get_options()->RecommendedVersions); - - format_iso_time(published, now); - - buf_len = 2048+strlen(recommended_versions); - - buf = tor_malloc(buf_len); - /* We'll be comparing against buf_len throughout the rest of the - function, though strictly speaking we shouldn't be able to exceed - it. This is C, after all, so we may as well check for buffer - overruns.*/ - - tor_snprintf(buf, buf_len, - "signed-directory\n" - "published %s\n" - "recommended-software %s\n" - "router-status %s\n" - "dir-signing-key\n%s\n", - published, recommended_versions, "", - identity_pkey); - - tor_free(recommended_versions); - tor_free(identity_pkey); - - cp = buf + strlen(buf); - *cp = '\0'; - - /* These multiple strlcat calls are inefficient, but dwarfed by the RSA - signature. */ - if (strlcat(buf, "directory-signature ", buf_len) >= buf_len) -goto truncated; - if (strlcat(buf, get_options()->Nickname, buf_len) >= buf_len) -goto truncated; - if (strlcat(buf, "\n", buf_len) >= buf_len) -goto truncated; - - if (router_get_dir_hash(buf,digest)) { -log_warn(LD_BUG,"couldn't compute digest"); -tor_free(buf); -return -1; - } - note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(buf,buf_len,digest,DIGEST_LEN, - private_key)<0) { -tor_free(buf); -return -1; - } - - *dir_out = buf; - return 0; - truncated: - log_warn(LD_BUG,"tried to exceed string length."); - tor_free(buf); - return -1; -} - // /* A set of functions to answer questions about how we'd like to behave diff --git a/src/or/dirserv.h b/src/or/dirserv.h index 9a9725a..3c914e9 100644 --- a/src/or/dirserv.h +++ b/src/or/dirserv.h @@ -47,8 +47,6 @@ enum was_router_added_t dirserv_add_descriptor(routerinfo_t *ri, void dirserv_set_router_is_running(routerinfo_t *router, time_t now); int list_server_status_v1(smartlist_t *routers, char **router_status_out, int for_controller); -int dirserv_dump_directory_to_string(char **dir_out, - crypto_pk_t *private_key); char *dirserv_get_flag_thresholds_line(void); void dirserv_compute_bridge_flag_thresholds(void); ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [translation/tor-messenger-fingerdtd] Update translations for tor-messenger-fingerdtd
commit 88390a939a5b56de115f88daa66ee6f690722d87 Author: Translation commit botDate: Mon May 30 19:46:22 2016 + Update translations for tor-messenger-fingerdtd --- pl/finger.dtd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pl/finger.dtd b/pl/finger.dtd index bd371ca..5943e49 100644 --- a/pl/finger.dtd +++ b/pl/finger.dtd @@ -12,4 +12,4 @@ - \ No newline at end of file + \ No newline at end of file ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [translation/tor-messenger-fingerdtd_completed] Update translations for tor-messenger-fingerdtd_completed
commit 53a12b4dccc1d3445c09978bfc493162e80311ec Author: Translation commit botDate: Mon May 30 19:46:25 2016 + Update translations for tor-messenger-fingerdtd_completed --- pl/finger.dtd | 7 ++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pl/finger.dtd b/pl/finger.dtd index fa58403..5943e49 100644 --- a/pl/finger.dtd +++ b/pl/finger.dtd @@ -7,4 +7,9 @@ - \ No newline at end of file + + + + + + \ No newline at end of file ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [tor/master] remove now-irrelevant XXX020 comments in configure.ac
commit 57bf8bb26357444219c07d57caa9260a75ff7681 Author: Nick MathewsonDate: Mon May 30 15:31:19 2016 -0400 remove now-irrelevant XXX020 comments in configure.ac They apply to ancient GCC versions and to an unknown set of configuration options. Notabug. --- configure.ac | 3 --- 1 file changed, 3 deletions(-) diff --git a/configure.ac b/configure.ac index 26f85b5..7259bf7 100644 --- a/configure.ac +++ b/configure.ac @@ -30,7 +30,6 @@ fi # the += operator on it in src/or/Makefile.am CPPFLAGS="$CPPFLAGS -I\${top_srcdir}/src/common" -#020 We should make these enabled or not, before 0.2.0.x-final AC_ARG_ENABLE(openbsd-malloc, AS_HELP_STRING(--enable-openbsd-malloc, [use malloc code from OpenBSD. Linux only])) AC_ARG_ENABLE(static-openssl, @@ -1710,7 +1709,6 @@ if test "x$enable_gcc_warnings_advisory" != "xno"; then if test "x$have_gcc42" = "xyes"; then # These warnings break gcc 4.0.2 and work on gcc 4.2 -# 020 See if any of these work with earlier versions. CFLAGS="$CFLAGS -Waddress -Wmissing-noreturn -Wstrict-overflow=1" # We used to use -Wstrict-overflow=5, but that breaks us heavily under 4.3. @@ -1723,7 +1721,6 @@ if test "x$enable_gcc_warnings_advisory" != "xno"; then if test "x$have_gcc43" = "xyes"; then # These warnings break gcc 4.2 and work on gcc 4.3 -# 020 See if any of these work with earlier versions. CFLAGS="$CFLAGS -Wextra -Warray-bounds" fi ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [webwml/master] Add new Tor Browser version: 6.0
commit 9f11ba3c13dd68089b942cb2dbd3b2804e8a3e3c Author: Nicolas VigierDate: Mon May 30 20:56:17 2016 +0200 Add new Tor Browser version: 6.0 --- include/versions.wmi | 22 +++--- projects/torbrowser/RecommendedTBBVersions | 4 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/include/versions.wmi b/include/versions.wmi index 750b575..e7c4b2c 100644 --- a/include/versions.wmi +++ b/include/versions.wmi @@ -4,27 +4,27 @@ 0.2.7.6 maint-5.5 -5.5.5 +6.0 6.0a5 -5.5.5 -2016-04-26 +6.0 +2016-05-30 6.0a5 2016-04-28 -5.5.5 -2016-04-26 -5.5.5 -2016-04-26 +6.0 +2016-05-30 +6.0 +2016-05-30 6.0a5 2016-04-28 6.0a5 2016-04-28 -5.5.5 -2016-04-26 -5.5.5 -2016-04-26 +6.0 +2016-05-30 +6.0 +2016-05-30 6.0a5 2016-04-28 diff --git a/projects/torbrowser/RecommendedTBBVersions b/projects/torbrowser/RecommendedTBBVersions index 61ac009..7b9dc5e 100644 --- a/projects/torbrowser/RecommendedTBBVersions +++ b/projects/torbrowser/RecommendedTBBVersions @@ -3,6 +3,10 @@ "5.5.5-Linux", "5.5.5-MacOS", "5.5.5-Windows", +"6.0", +"6.0-Linux", +"6.0-MacOS", +"6.0-Windows", "6.0a5", "6.0a5-Linux", "6.0a5-MacOS", ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Address feedback by @bassosimone
commit 2d5b7988c23697cd6af390c6ed649d69b7380dff Author: Arturo FilastòDate: Tue May 24 11:34:58 2016 +0200 Address feedback by @bassosimone * Distinguish between http-failure and http-diff * Update description of web_connectivity test * Various consistency and stylistic improvements --- ooni/nettests/blocking/web_connectivity.py | 23 --- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 4f5f4e4..3560cb0 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -68,8 +68,10 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): Web connectivity """ name = "Web connectivity" -description = ("Performs a HTTP GET request over Tor and one over the " - "local network and compares the two results.") +description = ("Identifies the reason for blocking of a given URL by " + "performing DNS resolution of the hostname, doing a TCP " + "connect to the resolved IPs and then fetching the page " + "and comparing all these results with those of a control.") author = "Arturo Filastò" version = "0.1.0" @@ -126,12 +128,12 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): if not l: continue # Skip comment lines -elif l.startswith('#'): +if l.startswith('#'): continue yield l +fh = open(filename) try: -fh = open(filename) line = fh.readline() # Detect the line of the citizenlab input file if line.startswith("url,"): @@ -140,7 +142,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): fh.seek(0) generator = simple_file_generator(fh) for i in generator: -if not i.startswith("http"): +if (not i.startswith("http://;) and +not i.startswith("https://;)): i = "http://{}/".format(i) yield i finally: @@ -200,7 +203,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): log.msg("* doing DNS query for {}".format(self.hostname)) return self.performALookup(self.hostname) -def tcp_connect(self, socket): +def experiment_tcp_connect(self, socket): log.msg("* connecting to {}".format(socket)) ip_address, port = socket.split(":") port = int(port) @@ -365,12 +368,10 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): experiment_http_failure is not None): blocking = 'tcp_ip' -# XXX we may want to have different codes for these two types of -# blocking elif (dns_consistent == True and tcp_connect == True and got_expected_web_page == False): -blocking = 'http' +blocking = 'http-diff' elif (dns_consistent == True and tcp_connect == True and @@ -379,7 +380,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): if experiment_http_failure == 'dns_lookup_error': blocking = 'dns' else: -blocking = 'http' +blocking = 'http-failure' elif (dns_consistent == False and (got_expected_web_page == False or @@ -425,7 +426,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): # STEALTH in here we should make changes to make the test more stealth dl = [] for socket in sockets: -dl.append(self.tcp_connect(socket)) +dl.append(self.experiment_tcp_connect(socket)) results = yield defer.DeferredList(dl) experiment_http = self.experiment_http_get_request() ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Various improvement related to backend_client
commit 24ea012b508526165b1a7d1eef1d76475bd313e6 Author: Arturo FilastòDate: Tue May 17 20:02:09 2016 +0200 Various improvement related to backend_client * Include web_connectivity in default generated deck * Fix serialisation of HTTP Headers * Support setting collector and bouncer in oonireport upload --- ooni/deck.py| 4 +++- ooni/deckgen/cli.py | 22 +++--- ooni/report/cli.py | 2 +- ooni/report/tool.py | 30 +- ooni/reporter.py| 3 ++- ooni/templates/httpt.py | 2 +- 6 files changed, 35 insertions(+), 28 deletions(-) diff --git a/ooni/deck.py b/ooni/deck.py index 007097e..404a4d0 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -136,10 +136,12 @@ class Deck(InputFile): net_test_loader = NetTestLoader(test['options']['subargs'], annotations=test['options'].get('annotations', {}), test_file=nettest_path) -if test['options']['collector']: +if test['options']['collector'] is not None: net_test_loader.collector = CollectorClient( test['options']['collector'] ) +if test['options']['bouncer'] is not None: +self.bouncer = test['options']['bouncer'] self.insert(net_test_loader) def insert(self, net_test_loader): diff --git a/ooni/deckgen/cli.py b/ooni/deckgen/cli.py index 2e0e976..a6046f3 100644 --- a/ooni/deckgen/cli.py +++ b/ooni/deckgen/cli.py @@ -27,6 +27,7 @@ class Options(usage.Options): "generate the deck."], ["collector", None, None, "Specify a custom collector to use when " "submitting reports"], +["bouncer", None, None, "Specify a custom bouncer to use"], ["output", "o", None, "Specify the directory where to write output."] ] @@ -44,7 +45,6 @@ class Deck(object): "annotations": None, "collector": None, -# XXX setting this is currently not supported "bouncer": None, "reportfile": None, @@ -56,13 +56,15 @@ class Deck(object): } } -def __init__(self, collector=None): +def __init__(self, collector=None, bouncer=None): self.deck_entries = [] self.collector = collector +self.bouncer = bouncer def add_test(self, test_file, subargs=[]): deck_entry = copy.deepcopy(self._base_entry) deck_entry['options']['collector'] = self.collector +deck_entry['options']['bouncer'] = self.bouncer deck_entry['options']['test_file'] = test_file deck_entry['options']['subargs'] = subargs self.deck_entries.append(deck_entry) @@ -89,24 +91,14 @@ def generate_deck(options): url_list_global = citizenlab_test_lists.generate_global_input( options['output'] ) -dns_servers = namebench_dns_servers.generate_country_input( -options['country-code'], -options['output'] -) -deck = Deck(collector=options['collector']) +deck = Deck(collector=options['collector'], bouncer=options['bouncer']) deck.add_test('manipulation/http_invalid_request_line') deck.add_test('manipulation/http_header_field_manipulation') if url_list_country is not None: -deck.add_test('blocking/http_requests', ['-f', url_list_country]) -deck.add_test('blocking/http_requests', ['-f', url_list_global]) - -if url_list_country is not None: -deck.add_test('blocking/dns_consistency', - ['-f', url_list_country, '-T', dns_servers]) -deck.add_test('blocking/dns_consistency', - ['-f', url_list_global, '-T', dns_servers]) +deck.add_test('blocking/web_connectivity', ['-f', url_list_country]) +deck.add_test('blocking/web_connectivity', ['-f', url_list_global]) if config.advanced.debug: deck.pprint() diff --git a/ooni/report/cli.py b/ooni/report/cli.py index c1b4a22..e52b022 100644 --- a/ooni/report/cli.py +++ b/ooni/report/cli.py @@ -20,7 +20,7 @@ class Options(usage.Options): "Specify the configuration file to use."], ["collector", "c", None, "Specify the collector to upload the result to."], -["bouncer", "b", "httpo://nkvphnp3p6agi5qq.onion", +["bouncer", "b", None, "Specify the bouncer to query for a collector."] ] diff --git a/ooni/report/tool.py b/ooni/report/tool.py index fd504a6..d80a8e0 100644 --- a/ooni/report/tool.py +++ b/ooni/report/tool.py @@ -9,12 +9,15 @@ from ooni.reporter import OONIBReporter, OONIBReportLog from ooni.utils import log from ooni.report import parser from ooni.settings import config -from ooni.backend_client import BouncerClient +from ooni.backend_client import BouncerClient, CollectorClient
[tor-commits] [ooni-probe/master] Expand the heuristics for detecting blockpages in HTTP response
commit 7f8021efd69d28beb034a7cd25c60ea2c0016bf6 Author: Arturo FilastòDate: Tue May 24 18:06:00 2016 +0200 Expand the heuristics for detecting blockpages in HTTP response * Extract the title of the response * Consider only common HTTP headers --- ooni/nettests/blocking/web_connectivity.py | 76 ++ ooni/utils/net.py | 39 +++ 2 files changed, 95 insertions(+), 20 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index e640a5f..8f048e1 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -1,7 +1,6 @@ # -*- encoding: utf-8 -*- import csv -import json from urlparse import urlparse from ipaddr import IPv4Address, AddressValueError @@ -20,7 +19,7 @@ from ooni.utils import log from ooni.backend_client import WebConnectivityClient -from ooni.utils.net import StringProducer, BodyReceiver +from ooni.utils.net import COMMON_SERVER_HEADERS, extract_title from ooni.templates import httpt, dnst from ooni.errors import failureToString @@ -50,6 +49,7 @@ class UsageOptions(usage.Options): ['url', 'u', None, 'Specify a single URL to test'], ['dns-discovery', 'd', 'whoami.akamai.net', 'Specify the dns discovery test helper'], ['backend', 'b', None, 'The web_consistency backend test helper'], +['retries', 'r', 1, 'Number of retries for the HTTP request'], ] @@ -158,6 +158,12 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): if not self.input: raise Exception("No input specified") +try: +self.localOptions['retries'] = int(self.localOptions['retries']) +except ValueError: +self.localOptions['retries'] = 2 + +self.report['retries'] = self.localOptions['retries'] self.report['client_resolver'] = self.resolverIp self.report['dns_consistency'] = None self.report['body_length_match'] = None @@ -188,7 +194,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): 'body_length': -1, 'failure': None, 'status_code': -1, -'headers': {} +'headers': {}, +'title': '' } } if isinstance(self.localOptions['backend'], dict): @@ -240,24 +247,36 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): ) self.report['control'] = self.control +@defer.inlineCallbacks def experiment_http_get_request(self): -return self.doRequest(self.input, headers=REQUEST_HEADERS) +retries = 0 +while True: +try: +result = yield self.doRequest(self.input, + headers=REQUEST_HEADERS) +break +except: +if self.localOptions['retries'] > retries: +raise +retries += 1 + +defer.returnValue(result) def compare_headers(self, experiment_http_response): -count = 0 control_headers_lower = {k.lower(): v for k, v in -self.report['control']['http_request']['headers'].items()} +self.report['control']['http_request']['headers'].items() +} +experiment_headers_lower = {k.lower(): v for k, v in +experiment_http_response.headers.getAllRawHeaders() +} -for header_name, header_value in \ -experiment_http_response.headers.getAllRawHeaders(): -try: -control_headers_lower[header_name.lower()] -except KeyError: -log.debug("Did not find the key {}".format(header_name)) -return False -count += 1 +uncommon_ctrl_headers = (set(control_headers_lower.keys()) - + set(COMMON_SERVER_HEADERS)) +uncommon_exp_headers = (set(experiment_headers_lower.keys()) - +set(COMMON_SERVER_HEADERS)) -return count == len(self.report['control']['http_request']['headers']) +return len(uncommon_ctrl_headers.intersection( +uncommon_exp_headers)) > 0 def compare_body_lengths(self, experiment_http_response): control_body_length = self.control['http_request']['body_length'] @@ -279,6 +298,17 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): else: return False +def compare_titles(self, experiment_http_response): +experiment_title = extract_title(experiment_http_response.body).strip() +control_title = self.control['http_request']['title'].strip() +first_exp_word = experiment_title.split(' ')[0] +first_ctrl_word = control_title.split(' ')[0] +if len(first_exp_word) < 5: +
[tor-commits] [ooni-probe/master] Don't require the bouncer key to be present
commit 156ce3f630eb58a061841dfa6d6f6f6cf6755d83 Author: Arturo FilastòDate: Tue May 17 20:24:15 2016 +0200 Don't require the bouncer key to be present --- ooni/deck.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/deck.py b/ooni/deck.py index 404a4d0..7a3b697 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -136,11 +136,11 @@ class Deck(InputFile): net_test_loader = NetTestLoader(test['options']['subargs'], annotations=test['options'].get('annotations', {}), test_file=nettest_path) -if test['options']['collector'] is not None: +if test['options'].get('collector', None) is not None: net_test_loader.collector = CollectorClient( test['options']['collector'] ) -if test['options']['bouncer'] is not None: +if test['options'].get('bouncer', None) is not None: self.bouncer = test['options']['bouncer'] self.insert(net_test_loader) ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Update to version 1.5.0
commit 5fdaa3ebe0928bbbc8cd8088d160cf1a40dd9730 Author: Arturo FilastòDate: Mon May 30 17:53:28 2016 +0200 Update to version 1.5.0 --- ChangeLog.rst| 10 ++ ooni/__init__.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 56dd061..5470570 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,6 +1,16 @@ Changelog = +v1.5.0 (Mon, 30 May 2016) +- +codename: The Big Wave + +* Implement web_connectivity test that measures for both DNS and HTTP + censorship. + +* Fix a regression bug that lead to Tor exit ip address not being included in + reports. + v1.4.2 (Fri, 29 Apr 2016) - diff --git a/ooni/__init__.py b/ooni/__init__.py index e02b430..59c0143 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- __author__ = "Open Observatory of Network Interference" -__version__ = "1.5.0.dev0" +__version__ = "1.5.0" # This is the version number of resources to be downloaded # when a release is made it should be aligned to __version__ __resources_version__ = "1.4.2" ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Match status codes only when the control status code is not an error
commit cd36a0d059b88810695a1b952ab54e853253f17e Author: Arturo FilastòDate: Wed May 25 12:35:22 2016 +0200 Match status codes only when the control status code is not an error --- ooni/nettests/blocking/web_connectivity.py | 13 +++-- 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 8f048e1..7692d70 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -317,10 +317,11 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['headers_match'] = \ self.compare_headers(experiment_http_response) -self.report['status_code_match'] = ( -experiment_http_response.code == -self.control['http_request']['status_code'] -) +if str(self.control['http_request']['status_code'])[0] != '5': +self.report['status_code_match'] = ( +self.control['http_request']['status_code'] == +experiment_http_response.code +) self.report['title_match'] = self.compare_titles(experiment_http_response) @@ -397,8 +398,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): got_expected_web_page = ( (self.report['body_length_match'] is True or self.report['headers_match'] is True or - self.report['title_match']) -and self.report['status_code_match'] is True + self.report['title_match'] is True) +and self.report['status_code_match'] is not False ) if (dns_consistent == True and tcp_connect == False and ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Add tor to the apt-get command.
commit 07eef0862220b7a1d7f651e2ce5511707cdc1d4a Author: Arturo FilastòDate: Sat May 21 14:07:49 2016 +0200 Add tor to the apt-get command. --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 5227cc6..1c2ec82 100644 --- a/README.rst +++ b/README.rst @@ -109,7 +109,7 @@ Make sure you have installed the following dependencies: On debian based systems this can generally be done by running:: -sudo apt-get install -y build-essential libdumbnet-dev libpcap-dev libgeoip-dev libffi-dev python-dev python-pip +sudo apt-get install -y build-essential libdumbnet-dev libpcap-dev libgeoip-dev libffi-dev python-dev python-pip tor Then you should be able to install ooniprobe by running:: ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Address feedback by @bassosimone
commit 8e7199d85c877353b83b3da98936c602d4c4a5b4 Author: Arturo FilastòDate: Mon May 30 10:42:29 2016 +0200 Address feedback by @bassosimone * Test the next words if the word is less than 5 chars in web_connectivity test. * Move the charset detection charset into the correct module and change the unittest accordingly. * Include ooni.common in setup.py * Various code style fixes --- ooni/backend_client.py | 6 +- ooni/common/http_utils.py | 2 +- ooni/nettests/blocking/web_connectivity.py | 21 - ooni/templates/httpt.py| 4 ooni/tests/test_common.py | 19 +++ ooni/tests/test_templates.py | 16 ooni/utils/net.py | 1 - setup.py | 1 + 8 files changed, 34 insertions(+), 36 deletions(-) diff --git a/ooni/backend_client.py b/ooni/backend_client.py index d4c463e..0e85dd7 100644 --- a/ooni/backend_client.py +++ b/ooni/backend_client.py @@ -207,9 +207,7 @@ class CollectorClient(OONIBClient): @d.addErrback def err(failure): failure.trap(Error) -if failure.value.status == '404': -return True -return False +return failure.value.status == '404' return d @@ -345,8 +343,6 @@ class CollectorClient(OONIBClient): class WebConnectivityClient(OONIBClient): def isReachable(self): -# XXX maybe in the future we can have a dedicated API endpoint to -# test the reachability of the collector. d = self.queryBackend('GET', '/status') @d.addCallback diff --git a/ooni/common/http_utils.py b/ooni/common/http_utils.py index 6d636d5..57c3a15 100644 --- a/ooni/common/http_utils.py +++ b/ooni/common/http_utils.py @@ -2,7 +2,7 @@ import re import codecs from base64 import b64encode -META_CHARSET_REGEXP = re.compile(']*?charset\s*=[\s"\']*([^\s"\'/>]*)') +META_CHARSET_REGEXP = re.compile(']*?charset\s*=[\s"\']*([^\s"\'/>!;]+)') def representBody(body): if not body: diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 655bf76..b4ccff1 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -42,10 +42,9 @@ class UsageOptions(usage.Options): def is_public_ipv4_address(address): try: ip_address = IPv4Address(address) -if not any([ip_address.is_private, -ip_address.is_loopback]): -return True -return False +return not any( +[ip_address.is_private, ip_address.is_loopback] +) except AddressValueError: return None @@ -293,13 +292,17 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): def compare_titles(self, experiment_http_response): experiment_title = extractTitle(experiment_http_response.body).strip() control_title = self.control['http_request']['title'].strip() -first_exp_word = experiment_title.split(' ')[0] -first_ctrl_word = control_title.split(' ')[0] -if len(first_exp_word) < 5: + +control_words = control_title.split(' ') +for exp_word, idx in enumerate(experiment_title.split(' ')): # We don't consider to match words that are shorter than 5 # characters (5 is the average word length for english) -return False -return (first_ctrl_word.lower() == first_exp_word.lower()) +if len(exp_word) < 5: +continue +try: +return control_words[idx].lower() == exp_word.lower() +except IndexError: +return False def compare_http_experiments(self, experiment_http_response): diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index f8ea941..3f7e77d 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -1,5 +1,3 @@ -import re -import codecs import random from txtorcon.interface import StreamListenerMixin @@ -22,8 +20,6 @@ from ooni.common.txextra import FixedRedirectAgent, TrueHeadersAgent from ooni.common.http_utils import representBody from ooni.errors import handleAllFailures -META_CHARSET_REGEXP = re.compile(']*?charset\s*=[\s"\']*([^\s"\'/>!;]+)') - class InvalidSocksProxyOption(Exception): pass diff --git a/ooni/tests/test_common.py b/ooni/tests/test_common.py new file mode 100644 index 000..1cd77cf --- /dev/null +++ b/ooni/tests/test_common.py @@ -0,0 +1,19 @@ +from twisted.trial import unittest +from ooni.common.http_utils import META_CHARSET_REGEXP + +class TestHTTPUtils(unittest.TestCase): +def test_charset_detection(self): +no_charset_html
[tor-commits] [ooni-probe/master] Merge pull request #514 from TheTorProject/feature/web_connectivity
commit 1c030e17e5390e3417d7582a5b3613c8696e53b1 Merge: a62a614 c046653 Author: Arturo FilastòDate: Mon May 30 17:39:59 2016 +0200 Merge pull request #514 from TheTorProject/feature/web_connectivity Feature/web connectivity bin/oonireport | 2 +- data/ooniprobe.conf.sample | 2 +- ooni/backend_client.py | 365 ++ ooni/common/__init__.py| 10 + ooni/common/http_utils.py | 55 ++ ooni/common/tcp_utils.py | 10 + ooni/common/txextra.py | 202 ooni/deck.py | 234 ++--- ooni/deckgen/cli.py| 22 +- ooni/director.py | 51 +- ooni/errors.py | 42 +- ooni/geoip.py | 25 +- ooni/nettest.py| 23 +- ooni/nettests/blocking/web_connectivity.py | 554 + .../manipulation/http_header_field_manipulation.py | 2 +- ooni/oonibclient.py| 221 ooni/oonicli.py| 43 +- ooni/report/cli.py | 2 +- ooni/report/tool.py| 32 +- ooni/reporter.py | 270 +++--- ooni/settings.py | 2 +- ooni/tasks.py | 5 +- ooni/templates/httpt.py| 95 ++-- ooni/templates/scapyt.py | 8 +- ooni/tests/mocks.py| 38 +- ooni/tests/test_common.py | 19 + ooni/tests/test_deck.py| 25 +- ooni/tests/test_director.py| 2 + ooni/tests/test_nettest.py | 13 +- ooni/tests/test_onion.py | 16 + ooni/tests/test_oonibclient.py | 47 +- ooni/tests/test_oonicli.py | 10 +- ooni/tests/test_reporter.py| 32 +- ooni/tests/test_templates.py | 14 +- ooni/tests/test_trueheaders.py | 2 +- ooni/tests/test_utils.py | 34 +- ooni/utils/__init__.py | 64 +-- ooni/utils/log.py | 2 +- ooni/utils/net.py | 54 +- ooni/utils/onion.py| 8 + ooni/utils/socks.py| 25 + ooni/utils/trueheaders.py | 181 --- setup.py | 1 + 43 files changed, 1858 insertions(+), 1006 deletions(-) ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Inject the correct url when following redirects
commit 6380468033cd7cca542184ebe8d2b4148aaec2b7 Author: Arturo FilastòDate: Tue May 24 18:25:40 2016 +0200 Inject the correct url when following redirects --- ooni/templates/httpt.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index ca0306b..6e0154c 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -196,6 +196,9 @@ class HTTPTest(NetTestCase): if (config.privacy.includeip is False and config.probe_ip.address is not None and (isinstance(response_body, str) or isinstance(response_body, unicode))): response_body = response_body.replace(config.probe_ip.address, "[REDACTED]") +if (getattr(response, 'request', None) and +getattr(response.request, 'absoluteURI', None)): +session['request']['url'] = response.request.absoluteURI session['response'] = { 'headers': _representHeaders(response.headers), 'body': response_body, ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Write a msg when we perform the control lookup
commit c046653fbd63ff6beb35aa1d87d0bbe5e0014fef Author: Arturo FilastòDate: Mon May 30 15:18:31 2016 +0200 Write a msg when we perform the control lookup Be more lax when detecting header key similarity --- ooni/nettests/blocking/web_connectivity.py | 5 + 1 file changed, 5 insertions(+) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index b4ccff1..0dba239 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -229,6 +229,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): @defer.inlineCallbacks def control_request(self, sockets): +log.msg("* performing control request with backend") self.control = yield self.web_connectivity_client.control( http_request=self.input, tcp_connect=sockets @@ -261,6 +262,10 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): experiment_http_response.headers.getAllRawHeaders() } +if (set(control_headers_lower.keys()) == +set(experiment_headers_lower.keys())): +return True + uncommon_ctrl_headers = (set(control_headers_lower.keys()) - set(COMMON_SERVER_HEADERS)) uncommon_exp_headers = (set(experiment_headers_lower.keys()) - ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Add support for Gzip Decoding in web connectivity test
commit f3cb68e06118ab7c20de9559404fae76715a5500 Author: Arturo FilastòDate: Mon May 23 12:10:32 2016 +0200 Add support for Gzip Decoding in web connectivity test --- ooni/nettests/blocking/web_connectivity.py | 36 +- ooni/templates/httpt.py| 14 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 97cde60..f68b5aa 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -6,6 +6,7 @@ from urlparse import urlparse from ipaddr import IPv4Address, AddressValueError +from twisted.web.client import GzipDecoder from twisted.internet import reactor from twisted.internet.protocol import Factory, Protocol from twisted.internet.endpoints import TCP4ClientEndpoint @@ -72,6 +73,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): author = "Arturo Filastò" version = "0.1.0" +contentDecoders = [('gzip', GzipDecoder)] + usageOptions = UsageOptions inputFile = [ @@ -156,6 +159,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['dns_consistency'] = None self.report['body_length_match'] = None self.report['headers_match'] = None +self.report['status_code_match'] = None self.report['accessible'] = None self.report['blocking'] = None @@ -177,8 +181,9 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): 'ips': [] }, 'http_request': { -'body_length': None, -'failure': True, +'body_length': -1, +'failure': None, +'status_code': -1, 'headers': {} } } @@ -270,6 +275,19 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): else: return False +def compare_http_experiments(self, experiment_http_response): + +self.report['body_length_match'] = \ +self.compare_body_lengths(experiment_http_response) + +self.report['headers_match'] = \ +self.compare_headers(experiment_http_response) + +self.report['status_code_match'] = ( +experiment_http_response.code == +self.control['http_request']['status_code'] +) + def compare_dns_experiments(self, experiment_dns_answers): if self.control['dns']['failure'] is not None and \ self.control['dns']['failure'] == self.report['dns_experiment_failure']: @@ -319,7 +337,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): def determine_blocking(self, experiment_http_response, experiment_dns_answers): blocking = False -control_http_failure = self.report['control']['http_request']['failure'] +control_http_failure = self.control['http_request']['failure'] if control_http_failure is not None: control_http_failure = control_http_failure.split(" ")[0] @@ -328,10 +346,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): experiment_http_failure = experiment_http_failure.split(" ")[0] if (experiment_http_failure is None and control_http_failure is None): -self.report['body_length_match'] = self.compare_body_lengths( -experiment_http_response) -self.report['headers_match'] = self.compare_headers( -experiment_http_response) +self.compare_http_experiments(experiment_http_response) dns_consistent = self.compare_dns_experiments(experiment_dns_answers) if dns_consistent is True: @@ -341,7 +356,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): tcp_connect = self.compare_tcp_experiments() got_expected_web_page = (self.report['body_length_match'] or - self.report['headers_match']) + self.report['headers_match']) and \ +self.report['status_code_match'] if (dns_consistent == True and tcp_connect == False and experiment_http_failure is not None): @@ -488,7 +504,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): for reason, urls in summary['blocked'].items(): log.msg("") -log.msg("URLS blocked due to {}".format(reason)) -log.msg(""+'-'*len(reason)) +log.msg("URLS possibly blocked due to {}".format(reason)) +log.msg("-"+'-'*len(reason)) for url in urls: log.msg("* {}".format(url)) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 7cbfd1d..ca0306b 100644 --- a/ooni/templates/httpt.py +++
[tor-commits] [ooni-probe/master] Share code from ooni-backend via the ooni.common module
commit 3aa17317012aa0c85cff77d6942249f0bd68f8e6 Author: Arturo FilastòDate: Sat May 28 15:45:05 2016 +0200 Share code from ooni-backend via the ooni.common module --- ooni/backend_client.py | 2 +- ooni/common/__init__.py| 10 + ooni/common/http_utils.py | 55 + ooni/common/tcp_utils.py | 10 + ooni/common/txextra.py | 202 + ooni/nettests/blocking/web_connectivity.py | 25 +-- .../manipulation/http_header_field_manipulation.py | 2 +- ooni/templates/httpt.py| 42 +--- ooni/templates/scapyt.py | 8 +- ooni/tests/test_trueheaders.py | 2 +- ooni/utils/__init__.py | 8 - ooni/utils/net.py | 8 - ooni/utils/socks.py| 25 +++ ooni/utils/trueheaders.py | 241 - 14 files changed, 322 insertions(+), 318 deletions(-) diff --git a/ooni/backend_client.py b/ooni/backend_client.py index de01ceb..d4c463e 100644 --- a/ooni/backend_client.py +++ b/ooni/backend_client.py @@ -16,7 +16,7 @@ from ooni import errors as e from ooni.settings import config from ooni.utils import log, onion from ooni.utils.net import BodyReceiver, StringProducer, Downloader -from ooni.utils.trueheaders import TrueHeadersSOCKS5Agent +from ooni.utils.socks import TrueHeadersSOCKS5Agent class OONIBClient(object): diff --git a/ooni/common/__init__.py b/ooni/common/__init__.py new file mode 100644 index 000..7f6cf73 --- /dev/null +++ b/ooni/common/__init__.py @@ -0,0 +1,10 @@ +""" +This modules contains functionality that is shared amongst ooni-probe and +ooni-backend. If the code in here starts growing too much I think it would +make sense to either: + + * Make the code in here into it's own package that is imported by + ooni-probe and ooni-backend. + + * Merge ooniprobe with oonibackend. +""" diff --git a/ooni/common/http_utils.py b/ooni/common/http_utils.py new file mode 100644 index 000..6d636d5 --- /dev/null +++ b/ooni/common/http_utils.py @@ -0,0 +1,55 @@ +import re +import codecs +from base64 import b64encode + +META_CHARSET_REGEXP = re.compile(']*?charset\s*=[\s"\']*([^\s"\'/>]*)') + +def representBody(body): +if not body: +return body +# XXX perhaps add support for decoding gzip in the future. +body = body.replace('\0', '') +decoded = False +charsets = ['ascii', 'utf-8'] + +# If we are able to detect the charset of body from the meta tag +# try to decode using that one first +charset = META_CHARSET_REGEXP.search(body, re.IGNORECASE) +if charset: +try: +encoding = charset.group(1).lower() +codecs.lookup(encoding) +charsets.insert(0, encoding) +except (LookupError, IndexError): +# Skip invalid codecs and partial regexp match +pass + +for encoding in charsets: +try: +body = unicode(body, encoding) +decoded = True +break +except UnicodeDecodeError: +pass +if not decoded: +body = { +'data': b64encode(body), +'format': 'base64' +} +return body + +TITLE_REGEXP = re.compile("(.*?)", re.IGNORECASE | re.DOTALL) + +def extractTitle(body): +m = TITLE_REGEXP.search(body, re.IGNORECASE | re.DOTALL) +if m: +return unicode(m.group(1), errors='ignore') +return '' + +REQUEST_HEADERS = { +'User-Agent': ['Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, ' + 'like Gecko) Chrome/47.0.2526.106 Safari/537.36'], +'Accept-Language': ['en-US;q=0.8,en;q=0.5'], +'Accept': ['text/html,application/xhtml+xml,application/xml;q=0.9,' + '*/*;q=0.8'] +} diff --git a/ooni/common/tcp_utils.py b/ooni/common/tcp_utils.py new file mode 100644 index 000..7b7a8a4 --- /dev/null +++ b/ooni/common/tcp_utils.py @@ -0,0 +1,10 @@ +from twisted.internet.protocol import Factory, Protocol + +class TCPConnectProtocol(Protocol): +def connectionMade(self): +self.transport.loseConnection() + +class TCPConnectFactory(Factory): +noisy = False +def buildProtocol(self, addr): +return TCPConnectProtocol() diff --git a/ooni/common/txextra.py b/ooni/common/txextra.py new file mode 100644 index 000..7a84592 --- /dev/null +++ b/ooni/common/txextra.py @@ -0,0 +1,202 @@ +import itertools +from copy import copy + +from twisted.web.http_headers import Headers +from twisted.web import error + +from twisted.web.client import BrowserLikeRedirectAgent +from twisted.web._newclient import ResponseFailed +from twisted.web._newclient import HTTPClientParser, ParseError +from twisted.python.failure import Failure +
[tor-commits] [ooni-probe/master] Add a timeout to the web_connectivity test to avoid the test timing out before it's done
commit a21ff3c777fb245096bc850532cf08ee0c4893d5 Author: Arturo FilastòDate: Wed May 25 15:28:20 2016 +0200 Add a timeout to the web_connectivity test to avoid the test timing out before it's done --- ooni/nettests/blocking/web_connectivity.py | 18 ++ 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 7692d70..41f6c2a 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -50,6 +50,7 @@ class UsageOptions(usage.Options): ['dns-discovery', 'd', 'whoami.akamai.net', 'Specify the dns discovery test helper'], ['backend', 'b', None, 'The web_consistency backend test helper'], ['retries', 'r', 1, 'Number of retries for the HTTP request'], +['timeout', 't', 240, 'Total timeout for this test'], ] @@ -163,6 +164,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): except ValueError: self.localOptions['retries'] = 2 +self.timeout = int(self.localOptions['timeout']) + self.report['retries'] = self.localOptions['retries'] self.report['client_resolver'] = self.resolverIp self.report['dns_consistency'] = None @@ -237,6 +240,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): result['status']['success'] = False result['status']['failure'] = failureToString(failure) self.report['tcp_connect'].append(result) + return d @defer.inlineCallbacks @@ -249,6 +253,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): @defer.inlineCallbacks def experiment_http_get_request(self): +log.msg("* doing HTTP(s) request {}".format(self.input)) retries = 0 while True: try: @@ -256,8 +261,10 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): headers=REQUEST_HEADERS) break except: -if self.localOptions['retries'] > retries: +if retries > self.localOptions['retries']: +log.debug("Finished all the allowed retries") raise +log.debug("Re-running HTTP request") retries += 1 defer.returnValue(result) @@ -470,15 +477,18 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): experiment_http = self.experiment_http_get_request() @experiment_http.addErrback def http_experiment_err(failure): -self.report['http_experiment_failure'] = failureToString(failure) +failure_string = failureToString(failure) +log.err("Failed to perform HTTP request %s" % failure_string) +self.report['http_experiment_failure'] = failure_string experiment_http_response = yield experiment_http control_request = self.control_request(sockets) @control_request.addErrback def control_err(failure): -log.err("Failed to perform control lookup") -self.report['control_failure'] = failureToString(failure) +failure_string = failureToString(failure) +log.err("Failed to perform control lookup: %s" % failure_string) +self.report['control_failure'] = failure_string yield control_request ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Increase the default timeout for the vanilla tor test to 300 seconds (#512)
commit a62a614de8839aee9341ad5663598cfcdd0bb668 Author: Arturo FilastòDate: Mon May 23 14:57:18 2016 +0200 Increase the default timeout for the vanilla tor test to 300 seconds (#512) This closes #508 --- ooni/nettests/blocking/vanilla_tor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/nettests/blocking/vanilla_tor.py b/ooni/nettests/blocking/vanilla_tor.py index 19c4805..7ed050f 100644 --- a/ooni/nettests/blocking/vanilla_tor.py +++ b/ooni/nettests/blocking/vanilla_tor.py @@ -18,7 +18,7 @@ class TorIsNotInstalled(Exception): class UsageOptions(usage.Options): optParameters = [ -['timeout', 't', 200, +['timeout', 't', 300, 'Specify the timeout after which to consider ' 'the Tor bootstrapping process to have failed.'], ] ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Clean up the output of ooniprobe
commit a1cf2789966b9550e3a044a0d3df7393328a7dee Author: Arturo FilastòDate: Tue May 17 17:36:47 2016 +0200 Clean up the output of ooniprobe * Make OneShotFactory not be noisy * Don't log an error when we are just connecting to a backend to check if it's up. --- ooni/backend_client.py | 2 +- ooni/utils/net.py | 24 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/ooni/backend_client.py b/ooni/backend_client.py index ae376e4..3b551d4 100644 --- a/ooni/backend_client.py +++ b/ooni/backend_client.py @@ -143,7 +143,7 @@ class OONIBClient(object): except ValueError: raise e.get_error(None) if 'error' in response: -log.err("Got this backend error message %s" % response) +log.debug("Got this backend error message %s" % response) raise e.get_error(response['error']) return response diff --git a/ooni/utils/net.py b/ooni/utils/net.py index 1f87101..ad5454e 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -3,21 +3,21 @@ import socket from random import randint from zope.interface import implements -from twisted.internet import protocol, defer +from twisted.internet import defer +from twisted.internet.protocol import Factory, Protocol from twisted.web.iweb import IBodyProducer from scapy.config import conf from ooni.errors import IfaceError -try: -from twisted.internet.endpoints import connectProtocol -except ImportError: -def connectProtocol(endpoint, protocol): -class OneShotFactory(protocol.Factory): -def buildProtocol(self, addr): -return protocol -return endpoint.connect(OneShotFactory()) +# This is our own connectProtocol to avoid noisy twisted cluttering our logs +def connectProtocol(endpoint, protocol): +class OneShotFactory(Factory): +noisy = False +def buildProtocol(self, addr): +return protocol +return endpoint.connect(OneShotFactory()) # if sys.platform.system() == 'Windows': # import _winreg as winreg @@ -68,7 +68,7 @@ class StringProducer(object): pass -class BodyReceiver(protocol.Protocol): +class BodyReceiver(Protocol): def __init__(self, finished, content_length=None, body_processor=None): self.finished = finished self.data = "" @@ -92,7 +92,7 @@ class BodyReceiver(protocol.Protocol): self.finished.errback(exc) -class Downloader(protocol.Protocol): +class Downloader(Protocol): def __init__(self, download_path, finished, content_length=None): self.finished = finished @@ -113,7 +113,7 @@ class Downloader(protocol.Protocol): self.finished.callback(None) -class ConnectAndCloseProtocol(protocol.Protocol): +class ConnectAndCloseProtocol(Protocol): def connectionMade(self): self.transport.loseConnection() ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Use .dev0 to indicate master branch to conform to PEP-440 (#507)
commit 2af0f7ebdc669926b21b3a1bb27712bc54d04ecb Author: Arturo FilastòDate: Fri May 20 17:55:28 2016 +0200 Use .dev0 to indicate master branch to conform to PEP-440 (#507) --- ooni/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/__init__.py b/ooni/__init__.py index 15e4278..e02b430 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- __author__ = "Open Observatory of Network Interference" -__version__ = "1.5.0-master" +__version__ = "1.5.0.dev0" # This is the version number of resources to be downloaded # when a release is made it should be aligned to __version__ __resources_version__ = "1.4.2" ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Compare CNAME responses as well
commit d7a7d0905862b7d75aaee20f32f351f1a66bb926 Author: Arturo FilastòDate: Tue May 24 12:04:36 2016 +0200 Compare CNAME responses as well --- ooni/nettests/blocking/web_connectivity.py | 19 ++- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 3560cb0..e640a5f 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -181,7 +181,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.control = { 'tcp_connect': {}, 'dns': { -'ips': [] +'addrs': [], +'failure': None, }, 'http_request': { 'body_length': -1, @@ -297,23 +298,23 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['dns_consistency'] = 'consistent' return True -control_ips = set(self.control['dns']['ips']) -experiment_ips = set(experiment_dns_answers) +control_addrs = set(self.control['dns']['addrs']) +experiment_addrs = set(experiment_dns_answers) -if control_ips == experiment_ips: +if control_addrs == experiment_addrs: return True -for experiment_ip in experiment_ips: -if is_public_ipv4_address(experiment_ip) is False: +for experiment_addr in experiment_addrs: +if is_public_ipv4_address(experiment_addr) is False: return False -if len(control_ips.intersection(experiment_ips)) > 0: +if len(control_addrs.intersection(experiment_addrs)) > 0: return True experiment_asns = set(map(lambda x: geoip.IPToLocation(x)['asn'], - experiment_ips)) + experiment_addrs)) control_asns = set(map(lambda x: geoip.IPToLocation(x)['asn'], - control_ips)) + control_addrs)) # Remove the instance of AS0 when we fail to find the ASN control_asns.discard('AS0') ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Merge pull request #511 from TheTorProject/fix/exit_ip
commit e2de248c9e1b778dabe345eb3d9a659ec6e88cc2 Merge: 07eef08 3ce772e Author: anadahzDate: Sat May 21 20:22:36 2016 + Merge pull request #511 from TheTorProject/fix/exit_ip Fix regression bug that lead to exit_ip and exit_name not being included ooni/templates/httpt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Ensure the input key is properly set
commit c8df2379870e9210c4fe20541b5f6ca4f65e1cbb Author: Arturo FilastòDate: Mon May 23 13:12:36 2016 +0200 Ensure the input key is properly set * Improve readability of the blockpage detection check --- ooni/nettests/blocking/web_connectivity.py | 8 +--- ooni/tasks.py | 5 ++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index f68b5aa..4f5f4e4 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -355,9 +355,11 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['dns_consistency'] = 'inconsistent' tcp_connect = self.compare_tcp_experiments() -got_expected_web_page = (self.report['body_length_match'] or - self.report['headers_match']) and \ -self.report['status_code_match'] +got_expected_web_page = ( +(self.report['body_length_match'] is True or + self.report['headers_match'] is True) +and self.report['status_code_match'] is True +) if (dns_consistent == True and tcp_connect == False and experiment_http_failure is not None): diff --git a/ooni/tasks.py b/ooni/tasks.py index 72e211f..925089b 100644 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@ -113,10 +113,9 @@ class Measurement(TaskWithTimeout): if not hasattr(self.testInstance, '_start_time'): self.testInstance._start_time = time.time() -if 'input' not in self.testInstance.report.keys(): -self.testInstance.report['input'] = test_input - self.testInstance.setUp() +if 'input' not in self.testInstance.report.keys(): +self.testInstance.report['input'] = self.testInstance.input self.netTestMethod = getattr(self.testInstance, test_method) ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Refactor code for setting of tor port
commit 1e3954bb26b2ac94b968d3e60cbf7ef75bb4c9b9 Author: Arturo FilastòDate: Fri May 13 15:33:00 2016 +0200 Refactor code for setting of tor port --- ooni/director.py | 22 ++ 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index 82d0e85..e6f864e 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -345,11 +345,13 @@ class Director(object): log.msg("%d%%: %s" % (prog, summary)) tor_config = TorConfig() -if config.tor.control_port: -tor_config.ControlPort = config.tor.control_port +if config.tor.control_port is None: +config.tor.control_port = int(randomFreePort()) +if config.tor.socks_port is None: +config.tor.socks_port = int(randomFreePort()) -if config.tor.socks_port: -tor_config.SocksPort = config.tor.socks_port +tor_config.ControlPort = config.tor.control_port +tor_config.SocksPort = config.tor.socks_port if config.tor.data_dir: data_dir = os.path.expanduser(config.tor.data_dir) @@ -384,18 +386,6 @@ class Director(object): tor_config.User = pwd.getpwuid(os.geteuid()).pw_name tor_config.save() - -if not hasattr(tor_config, 'ControlPort'): -control_port = int(randomFreePort()) -tor_config.ControlPort = control_port -config.tor.control_port = control_port - -if not hasattr(tor_config, 'SocksPort'): -socks_port = int(randomFreePort()) -tor_config.SocksPort = socks_port -config.tor.socks_port = socks_port - -tor_config.save() log.debug("Setting control port as %s" % tor_config.ControlPort) log.debug("Setting SOCKS port as %s" % tor_config.SocksPort) ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Fix regression bug that lead to exit_ip and exit_name not being included
commit 3ce772e21ca395993d3ad2878cc394a768f5be79 Author: Arturo FilastòDate: Sat May 21 19:33:42 2016 +0200 Fix regression bug that lead to exit_ip and exit_name not being included This closes #509 --- ooni/templates/httpt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index e608f8f..fe283a7 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -30,7 +30,7 @@ class StreamListener(StreamListenerMixin): def stream_succeeded(self, stream): host=self.request['url'].split('/')[2] try: -if stream.target_host == host and len(self.request['tor']) == 1: +if stream.target_host == host and self.request['tor']['exit_ip'] is None: self.request['tor']['exit_ip'] = stream.circuit.path[-1].ip self.request['tor']['exit_name'] = stream.circuit.path[-1].name config.tor_state.stream_listeners.remove(self) ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Change test for setupCollector() function
commit 8fd571f03319462babc1ca925cb147498d28f138 Author: Arturo FilastòDate: Wed May 18 16:10:07 2016 +0200 Change test for setupCollector() function --- ooni/tests/test_oonicli.py | 10 -- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py index 24877e8..4a58736 100644 --- a/ooni/tests/test_oonicli.py +++ b/ooni/tests/test_oonicli.py @@ -221,5 +221,11 @@ class TestOoniCli(ConfigTestCase): global_options = { 'collector': collector1 } -collector_address = setupCollector(global_options, collector2) -self.assertEqual(collector_address, collector1) +try: +collector_client = setupCollector(global_options, collector2) +self.assertEqual(collector_client.settings['address'], collector1) +self.assertEqual(collector_client.settings['type'], 'https') +except errors.CollectorUnsupported: +# Older versions of twisted will raise this. We could be more +# strict and do a check for older twisted versions in here. +pass ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Consider valid onion addresses also when they don't use the httpo:// schema
commit c32bd88b5b99ea7d3fc62c9088d3ac17770ba7df Author: Arturo FilastòDate: Tue May 24 11:18:10 2016 +0200 Consider valid onion addresses also when they don't use the httpo:// schema Based on feedback from @willscott * Fix invalid onion address in unittests --- ooni/backend_client.py | 16 ++-- ooni/deck.py | 4 ++-- ooni/tests/mocks.py | 6 +++--- ooni/tests/test_deck.py | 2 +- ooni/tests/test_onion.py | 16 ooni/utils/onion.py | 8 6 files changed, 40 insertions(+), 12 deletions(-) diff --git a/ooni/backend_client.py b/ooni/backend_client.py index 3b551d4..de01ceb 100644 --- a/ooni/backend_client.py +++ b/ooni/backend_client.py @@ -14,7 +14,7 @@ _twisted_14_0_2_version = Version('twisted', 14, 0, 2) from ooni import errors as e from ooni.settings import config -from ooni.utils import log +from ooni.utils import log, onion from ooni.utils.net import BodyReceiver, StringProducer, Downloader from ooni.utils.trueheaders import TrueHeadersSOCKS5Agent @@ -42,10 +42,10 @@ class OONIBClient(object): def _guessBackendType(self): if self.base_address is None: raise e.InvalidAddress -if self.base_address.startswith('https://'): -self.backend_type = 'https' -elif self.base_address.startswith('httpo://'): +if onion.is_onion_address(self.base_address): self.backend_type = 'onion' +elif self.base_address.startswith('https://'): +self.backend_type = 'https' elif self.base_address.startswith('http://'): self.backend_type = 'http' else: @@ -54,10 +54,14 @@ class OONIBClient(object): def _setupBaseAddress(self): parsed_address = urlparse(self.base_address) if self.backend_type == 'onion': -if not parsed_address.netloc.endswith(".onion"): +if not onion.is_onion_address(self.base_address): log.err("Invalid onion address.") raise e.InvalidAddress(self.base_address) -self.base_address = ("http://%s; % parsed_address.netloc) +if parsed_address.scheme in ('http', 'httpo'): +self.base_address = ("http://%s; % parsed_address.netloc) +else: +self.base_address = ("%s://%s" % (parsed_address.scheme, + parsed_address.netloc)) elif self.backend_type == 'http': self.base_address = ("http://%s; % parsed_address.netloc) elif self.backend_type in ('https', 'cloudfront'): diff --git a/ooni/deck.py b/ooni/deck.py index 7a3b697..383845c 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -4,7 +4,7 @@ from ooni.backend_client import CollectorClient, BouncerClient from ooni.backend_client import WebConnectivityClient from ooni.nettest import NetTestLoader from ooni.settings import config -from ooni.utils import log +from ooni.utils import log, onion from ooni import errors as e from twisted.python.filepath import FilePath @@ -175,7 +175,7 @@ class Deck(InputFile): https_addresses = [] plaintext_addresses = [] -if priority_address.startswith('httpo://'): +if onion.is_onion_address(priority_address): priority_address = { 'address': priority_address, 'type': 'onion' diff --git a/ooni/tests/mocks.py b/ooni/tests/mocks.py index f3f852f..db7b154 100644 --- a/ooni/tests/mocks.py +++ b/ooni/tests/mocks.py @@ -197,13 +197,13 @@ class MockBouncerClient(object): ret = { 'default': { 'address': '127.0.0.1', -'collector': 'httpo://thirteenchars1234.onion' +'collector': 'httpo://thirteenchars123.onion' } } for required_test_helper in required_test_helpers: ret[required_test_helper] = { 'address': '127.0.0.1', -'collector': 'httpo://thirteenchars1234.onion' +'collector': 'httpo://thirteenchars123.onion' } return defer.succeed(ret) @@ -221,7 +221,7 @@ class MockBouncerClient(object): 'name': net_test['name'], 'version': net_test['version'], 'input-hashes': net_test['input-hashes'], -'collector': 'httpo://thirteenchars1234.onion', +'collector': 'httpo://thirteenchars123.onion', 'test-helpers': test_helpers }) return defer.succeed(ret) diff --git a/ooni/tests/test_deck.py b/ooni/tests/test_deck.py index 8d415f0..fa9b0e8 100644 --- a/ooni/tests/test_deck.py +++ b/ooni/tests/test_deck.py @@ -160,7 +160,7 @@ class TestDeck(BaseTestCase): yield deck.lookupCollectorAndTestHelpers() self.assertEqual(deck.netTestLoaders[0].collector.settings['address'], -
[tor-commits] [ooni-probe/master] Use settings.Config.ooni_home as the data directory (to avoid broken paths when using sudo or setting environment vars)
commit edcb8379aa78d2e9fdeb039aa7e6fd82170fa2dd Author: Arturo FilastòDate: Mon Apr 18 18:46:33 2016 +0200 Use settings.Config.ooni_home as the data directory (to avoid broken paths when using sudo or setting environment vars) --- ooni/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/settings.py b/ooni/settings.py index 6e0f9a5..ffbf68e 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -58,7 +58,7 @@ class OConfig(object): @property def data_directory_candidates(self): dirs = [ -os.path.join(expanduser('~'+self.current_user), '.ooni'), +self.ooni_home, self.var_lib_path, self.usr_share_path, os.path.join(get_ooni_root(), '..', 'data'), ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Fix some issues with the rebase
commit 8a6e1a50fc5aa6fc9726b111c9edc9f44008f921 Author: Arturo FilastòDate: Thu May 5 19:30:29 2016 +0200 Fix some issues with the rebase --- ooni/nettest.py| 2 -- ooni/reporter.py | 6 +++--- ooni/tests/test_nettest.py | 9 ++--- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 67d3e27..8b9556e 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -572,8 +572,6 @@ class NetTest(object): test_class.inputs = yield defer.maybeDeferred( test_class().getInputProcessor ) -if not test_class.inputs: -test_class.inputs = [None] # Run the setupClass method yield defer.maybeDeferred( diff --git a/ooni/reporter.py b/ooni/reporter.py index aa6f5ee..6103c1e 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -646,12 +646,12 @@ class Report(object): created callback of the reporter whose report got created. """ if self.collector_address: -self.oonib_reporter = OONIBReporter(self.net_test_details, +self.oonib_reporter = OONIBReporter(self.test_details, self.collector_address) -self.net_test_details['report_id'] = yield self.open_oonib_reporter() +self.test_details['report_id'] = yield self.open_oonib_reporter() if not self.no_yamloo: -self.yaml_reporter = YAMLReporter(self.net_test_details, +self.yaml_reporter = YAMLReporter(self.test_details, report_filename=self.report_filename) self.report_filename = self.yaml_reporter.report_path if not self.oonib_reporter: diff --git a/ooni/tests/test_nettest.py b/ooni/tests/test_nettest.py index 660fd3e..da4969f 100644 --- a/ooni/tests/test_nettest.py +++ b/ooni/tests/test_nettest.py @@ -261,13 +261,8 @@ class TestNetTest(unittest.TestCase): nt = NetTest(ntl.getTestCases(), ntl.getTestDetails(), None) yield nt.initialize() -# XXX: if you use the same test_class twice you will have consumed all -# of its inputs! -tested = set([]) -for test_class, test_method in ntl.getTestCases(): -if test_class not in tested: -tested.update([test_class]) -self.assertEqual(len(list(test_class.inputs)), 10) +for test_class, test_methods in nt.testCases: +self.assertEqual(len(list(test_class.inputs)), 10) def test_setup_local_options_in_test_cases(self): ntl = NetTestLoader(dummyArgs) ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Add the redirected responses in inverted order
commit 32c463b766476a0db697d8c72264ed5683563fd2 Author: Arturo FilastòDate: Sun May 8 19:08:52 2016 +0200 Add the redirected responses in inverted order This means the first item in the list will be the final response, while the last will be the first response. --- ooni/templates/httpt.py | 11 ++- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 6b4c4b9..c58daf2 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -162,11 +162,6 @@ class HTTPTest(NetTestCase): failure (instance): An instance of :class:twisted.internet.failure.Failure """ -if response and response.previousResponse: -self.addToReport(request, response.previousResponse, - response_body=None, - failure_string=None) - log.debug("Adding %s to report" % request) request_headers = TrueHeaders(request['headers']) session = { @@ -199,6 +194,12 @@ class HTTPTest(NetTestCase): self.report['requests'].append(session) +if response and response.previousResponse: +self.addToReport(request, response.previousResponse, + response_body=None, + failure_string=None) + + def _processResponseBody(self, response_body, request, response, body_processor): log.debug("Processing response body") HTTPTest.addToReport(self, request, response, response_body) ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Patch twisted HTTPClientParser to accept HTTP response lines with a missing reason
commit c50f50cb0984cefabc82bd4f559062dbc893736c Author: Arturo FilastòDate: Sun May 8 19:09:46 2016 +0200 Patch twisted HTTPClientParser to accept HTTP response lines with a missing reason --- ooni/utils/trueheaders.py | 20 1 file changed, 20 insertions(+) diff --git a/ooni/utils/trueheaders.py b/ooni/utils/trueheaders.py index a5b2fe3..5f2b399 100644 --- a/ooni/utils/trueheaders.py +++ b/ooni/utils/trueheaders.py @@ -115,6 +115,26 @@ class HTTPClientParser(_newclient.HTTPClientParser): headers.addRawHeader(name, value) +def statusReceived(self, status): +parts = status.split(b' ', 2) +if len(parts) != 3: +# Here we add the extra missing part. +parts.append("XXX") + +try: +statusCode = int(parts[1]) +except ValueError: +raise _newclient.ParseError(u"non-integer status code", status) + +self.response = _newclient.Response._construct( +self.parseVersion(parts[0]), +statusCode, +parts[2], +self.headers, +self.transport, +self.request) + + class HTTP11ClientProtocol(_newclient.HTTP11ClientProtocol): def request(self, request): if self._state != 'QUIESCENT': ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Take also into consideration if the HTTP headers match in detecting blockpages.
commit 958850898457e78e7091dd2aa5193e9956d26a8f Author: Arturo FilastòDate: Mon May 9 15:56:35 2016 +0200 Take also into consideration if the HTTP headers match in detecting blockpages. This significantly brings down the false positive ratio. Though there are still some instances of sites being mis-reported as blocked. Examples of these are sites that have different web servers for sites in the various languages. --- ooni/nettests/blocking/web_connectivity.py | 25 + 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 08cae57..eb05835 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -347,6 +347,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['dns_consistency'] = 'inconsistent' tcp_connect = self.compare_tcp_experiments() +got_expected_web_page = (self.report['body_length_match'] or + self.report['headers_match']) if (dns_consistent == True and tcp_connect == False and experiment_http_failure is not None): @@ -354,23 +356,30 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): # XXX we may want to have different codes for these two types of # blocking -elif (dns_consistent == True and tcp_connect == True and - self.report['body_length_match'] == False): -blocking = 'http' -elif (dns_consistent == True and tcp_connect == True and -experiment_http_failure is not None and -control_http_failure is None): +elif (dns_consistent == True and + tcp_connect == True and + got_expected_web_page == False): blocking = 'http' +elif (dns_consistent == True and + tcp_connect == True and + experiment_http_failure is not None and + control_http_failure is None): +if experiment_http_failure == 'dns_lookup_error': +blocking = 'dns' +else: +blocking = 'http' + elif (dns_consistent == False and - (experiment_http_failure is not None or - self.report['body_length_match'] == False)): + (got_expected_web_page == False or +experiment_http_failure is not None)): blocking = 'dns' # This happens when the DNS resolution is injected, but the domain # doesn't have a valid record anymore or it resolves to an address # that is only accessible from within the country/network of the probe. elif (dns_consistent == False and + got_expected_web_page == False and (self.control['dns']['failure'] is not None or control_http_failure != experiment_http_failure)): blocking = 'dns' ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Ignore empty charset values
commit d80a68098dc4fd13a4963e26d6c4e5ed2ad31b43 Author: Arturo FilastòDate: Sun May 8 17:55:46 2016 +0200 Ignore empty charset values --- ooni/templates/httpt.py | 2 +- ooni/tests/test_templates.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 2a17f5b..51fba1a 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -17,7 +17,7 @@ from ooni.utils.net import StringProducer, userAgents from ooni.utils.trueheaders import TrueHeaders from ooni.errors import handleAllFailures -META_CHARSET_REGEXP = re.compile(']*?charset\s*=[\s"\']*([^\s"\'/>]*)') +META_CHARSET_REGEXP = re.compile(']*?charset\s*=[\s"\']*([^\s"\'/>]+)') class InvalidSocksProxyOption(Exception): pass diff --git a/ooni/tests/test_templates.py b/ooni/tests/test_templates.py index 5b2c77a..e8fe636 100644 --- a/ooni/tests/test_templates.py +++ b/ooni/tests/test_templates.py @@ -54,8 +54,10 @@ class TestHTTPT(unittest.TestCase): Foo """ with_charset_html = no_charset_html + '\n' +with_empty_charset = no_charset_html + '\n' self.assertEqual(httpt.META_CHARSET_REGEXP.search(no_charset_html), None) self.assertEqual(httpt.META_CHARSET_REGEXP.search(with_charset_html).group(1), 'iso-8859-1') +self.assertEqual(httpt.META_CHARSET_REGEXP.search(with_empty_charset), None) class TestDNST(unittest.TestCase): def setUp(self): ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Improve detection logic for web_connectivity
commit 4dd1542306a6cb3615cd5844728a689a4be7c998 Author: Arturo FilastòDate: Tue May 3 15:23:18 2016 +0200 Improve detection logic for web_connectivity * Fix printing of exception log --- bin/oonireport | 2 +- ooni/nettests/blocking/web_connectivity.py | 22 +++--- ooni/utils/log.py | 2 +- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/bin/oonireport b/bin/oonireport index 6a79f4c..9f59683 100755 --- a/bin/oonireport +++ b/bin/oonireport @@ -14,8 +14,8 @@ def failed(failure): r = failure.trap(exceptions.SystemExit, Exception) if r != exceptions.SystemExit: -log.err("Failed to run oonireport") log.exception(failure) +log.err("Failed to run oonireport") exitCode = 127 else: exitCode = failure.value.code diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 69fd002..08cae57 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -326,8 +326,15 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): def determine_blocking(self, experiment_http_response, experiment_dns_answers): blocking = False -if (self.report['http_experiment_failure'] is None and -self.report['control']['http_request']['failure'] is None): +control_http_failure = self.report['control']['http_request']['failure'] +if control_http_failure is not None: +control_http_failure = control_http_failure.split(" ")[0] + +experiment_http_failure = self.report['http_experiment_failure'] +if experiment_http_failure is not None: +experiment_http_failure = experiment_http_failure.split(" ")[0] + +if (experiment_http_failure is None and control_http_failure is None): self.report['body_length_match'] = self.compare_body_lengths( experiment_http_response) self.report['headers_match'] = self.compare_headers( @@ -340,8 +347,9 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['dns_consistency'] = 'inconsistent' tcp_connect = self.compare_tcp_experiments() + if (dns_consistent == True and tcp_connect == False and -self.report['http_experiment_failure'] is not None): +experiment_http_failure is not None): blocking = 'tcp_ip' # XXX we may want to have different codes for these two types of @@ -350,12 +358,12 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['body_length_match'] == False): blocking = 'http' elif (dns_consistent == True and tcp_connect == True and -self.report['http_experiment_failure'] is not None and -self.report['control']['http_request']['failure'] != self.report['http_experiment_failure']): +experiment_http_failure is not None and +control_http_failure is None): blocking = 'http' elif (dns_consistent == False and - (self.report['http_experiment_failure'] is not None or + (experiment_http_failure is not None or self.report['body_length_match'] == False)): blocking = 'dns' @@ -364,7 +372,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): # that is only accessible from within the country/network of the probe. elif (dns_consistent == False and (self.control['dns']['failure'] is not None or - self.control['http_request']['failure'] is not None)): + control_http_failure != experiment_http_failure)): blocking = 'dns' return blocking diff --git a/ooni/utils/log.py b/ooni/utils/log.py index f36c0ff..fd80594 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -111,7 +111,7 @@ def exception(error): or it can be a twisted.python.failure.Failure instance. """ if isinstance(error, Failure): -error.printTraceback() +error.printTraceback(sys.stdout) else: exc_type, exc_value, exc_traceback = sys.exc_info() traceback.print_exception(exc_type, exc_value, exc_traceback) ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Make the test-helper-alternate key work when speaking to backend
commit 2b238683d01d2d974a3743a226dd30f02371faf0 Author: Arturo FilastòDate: Fri May 13 15:31:58 2016 +0200 Make the test-helper-alternate key work when speaking to backend * Rename advanced option insecure_collector to insecure_backend * Refactor isReachable() to only return booleans * Bugfixing --- data/ooniprobe.conf.sample | 2 +- ooni/backend_client.py | 21 - ooni/deck.py | 70 +- ooni/errors.py | 11 + ooni/nettests/blocking/web_connectivity.py | 1 + ooni/oonicli.py| 4 +- 6 files changed, 45 insertions(+), 64 deletions(-) diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample index edc3f86..72a8e06 100644 --- a/data/ooniprobe.conf.sample +++ b/data/ooniprobe.conf.sample @@ -48,7 +48,7 @@ advanced: report_log_file: null inputs_dir: null decks_dir: null -insecure_collector: false +insecure_backend: false tor: #socks_port: 8801 #control_port: 8802 diff --git a/ooni/backend_client.py b/ooni/backend_client.py index c7de7f0..ae376e4 100644 --- a/ooni/backend_client.py +++ b/ooni/backend_client.py @@ -23,7 +23,7 @@ class OONIBClient(object): def __init__(self, address=None, settings={}): self.base_headers = {} self.backend_type = settings.get('type', None) -self.base_address = settings.get('address', address).encode('ascii') +self.base_address = settings.get('address', address) if self.backend_type is None: self._guessBackendType() @@ -62,6 +62,7 @@ class OONIBClient(object): self.base_address = ("http://%s; % parsed_address.netloc) elif self.backend_type in ('https', 'cloudfront'): self.base_address = ("https://%s; % parsed_address.netloc) +self.base_address = self.base_address.encode('ascii') def isSupported(self): if self.backend_type in ("https", "cloudfront"): @@ -70,10 +71,10 @@ class OONIBClient(object): "twisted > 14.0.2.") return False elif self.backend_type == "http": -if config.advanced.insecure_collector is not True: +if config.advanced.insecure_backend is not True: log.err("Plaintext backends are not supported. To " "enable at your own risk set " -"advanced->insecure_collector to true") +"advanced->insecure_backend to true") return False elif self.backend_type == "onion": # XXX add an extra check to ensure tor is running @@ -159,7 +160,7 @@ class OONIBClient(object): class BouncerClient(OONIBClient): def isReachable(self): -pass +return defer.succeed(True) @defer.inlineCallbacks def lookupTestCollector(self, net_tests): @@ -197,14 +198,14 @@ class CollectorClient(OONIBClient): def cb(_): # We should never be getting an acceptable response for a # request to an invalid path. -raise e.CollectorUnreachable +return False @d.addErrback def err(failure): failure.trap(Error) if failure.value.status == '404': return True -raise e.CollectorUnreachable +return False return d @@ -346,13 +347,13 @@ class WebConnectivityClient(OONIBClient): @d.addCallback def cb(result): -if result.get("status", None) is not "ok": -raise e.TestHelperUnreachable +if result.get("status", None) != "ok": +return False return True @d.addErrback def err(_): -raise e.TestHelperUnreachable +return False return d @@ -361,4 +362,4 @@ class WebConnectivityClient(OONIBClient): 'http_request': http_request, 'tcp_connect': tcp_connect } -self.queryBackend('POST', '/', query=request) +return self.queryBackend('POST', '/', query=request) diff --git a/ooni/deck.py b/ooni/deck.py index 24c7904..007097e 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -184,11 +184,10 @@ class Deck(InputFile): 'type': 'https' } elif priority_address.startswith('http://'): -if config.advanced.insecure_collector is True: -priority_address = { -'address': priority_address, -'type': 'http' -} +priority_address = { +'address': priority_address, +'type': 'http' +} else: raise e.InvalidOONIBCollectorAddress @@ -200,13 +199,13 @@ class Deck(InputFile): cloudfront_addresses +=
[tor-commits] [ooni-probe/master] Start adding support for HTTPS, Cloudfronted test helpers and collectors
commit d17873211da4bd6ec3c0d449ea1d62c2216d1996 Author: Arturo FilastòDate: Wed May 11 15:11:13 2016 +0200 Start adding support for HTTPS, Cloudfronted test helpers and collectors * Add routines to verify which collectors and test helpers are reachable * Merciless refactoring of the reporting logic * Remove dumb logic --- ooni/backend_client.py | 364 + ooni/deck.py | 164 +++-- ooni/director.py | 24 +- ooni/errors.py | 49 +++- ooni/nettests/blocking/web_connectivity.py | 36 ++- ooni/oonibclient.py| 232 -- ooni/oonicli.py| 35 +-- ooni/report/tool.py| 4 +- ooni/reporter.py | 220 + ooni/tests/mocks.py| 12 +- ooni/tests/test_deck.py| 9 +- ooni/tests/test_oonibclient.py | 47 ++-- ooni/tests/test_reporter.py| 32 ++- ooni/tests/test_utils.py | 34 +-- ooni/utils/__init__.py | 56 ++--- 15 files changed, 729 insertions(+), 589 deletions(-) diff --git a/ooni/backend_client.py b/ooni/backend_client.py new file mode 100644 index 000..c7de7f0 --- /dev/null +++ b/ooni/backend_client.py @@ -0,0 +1,364 @@ +import os +import json + +from urlparse import urljoin, urlparse + +from twisted.web.error import Error +from twisted.web.client import Agent, Headers +from twisted.internet import defer, reactor +from twisted.internet.endpoints import TCP4ClientEndpoint + +from twisted.python.versions import Version +from twisted import version as _twisted_version +_twisted_14_0_2_version = Version('twisted', 14, 0, 2) + +from ooni import errors as e +from ooni.settings import config +from ooni.utils import log +from ooni.utils.net import BodyReceiver, StringProducer, Downloader +from ooni.utils.trueheaders import TrueHeadersSOCKS5Agent + + +class OONIBClient(object): +def __init__(self, address=None, settings={}): +self.base_headers = {} +self.backend_type = settings.get('type', None) +self.base_address = settings.get('address', address).encode('ascii') + +if self.backend_type is None: +self._guessBackendType() +self.backend_type = self.backend_type.encode('ascii') + +if self.backend_type == 'cloudfront': +self.base_headers['Host'] = settings['front'].encode('ascii') + +self._setupBaseAddress() +self.settings = { +'type': self.backend_type, +'address': self.base_address, +'front': settings.get('front', '').encode('ascii') +} + +def _guessBackendType(self): +if self.base_address is None: +raise e.InvalidAddress +if self.base_address.startswith('https://'): +self.backend_type = 'https' +elif self.base_address.startswith('httpo://'): +self.backend_type = 'onion' +elif self.base_address.startswith('http://'): +self.backend_type = 'http' +else: +raise e.InvalidAddress + +def _setupBaseAddress(self): +parsed_address = urlparse(self.base_address) +if self.backend_type == 'onion': +if not parsed_address.netloc.endswith(".onion"): +log.err("Invalid onion address.") +raise e.InvalidAddress(self.base_address) +self.base_address = ("http://%s; % parsed_address.netloc) +elif self.backend_type == 'http': +self.base_address = ("http://%s; % parsed_address.netloc) +elif self.backend_type in ('https', 'cloudfront'): +self.base_address = ("https://%s; % parsed_address.netloc) + +def isSupported(self): +if self.backend_type in ("https", "cloudfront"): +if _twisted_version < _twisted_14_0_2_version: +log.err("HTTPS and cloudfronted backends require " +"twisted > 14.0.2.") +return False +elif self.backend_type == "http": +if config.advanced.insecure_collector is not True: +log.err("Plaintext backends are not supported. To " +"enable at your own risk set " +"advanced->insecure_collector to true") +return False +elif self.backend_type == "onion": +# XXX add an extra check to ensure tor is running +if not config.tor_state and config.tor.socks_port is None: +return False +return True + +def isReachable(self): +raise NotImplemented + +def _request(self, method, urn, genReceiver, bodyProducer=None, retries=3): +if self.backend_type == 'onion': +agent =
[tor-commits] [ooni-probe/master] Improvements to the output of the web_connectivity test
commit 6f0154b33f3528e3fdd78ca0f312e174d89bae33 Author: Arturo FilastòDate: Mon May 2 17:53:58 2016 +0200 Improvements to the output of the web_connectivity test --- ooni/nettests/blocking/web_connectivity.py | 82 ++ 1 file changed, 71 insertions(+), 11 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 28ee942..69fd002 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -1,5 +1,6 @@ # -*- encoding: utf-8 -*- +import csv import json from urlparse import urlparse @@ -102,6 +103,43 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): log.exception(exc) log.err("Failed to lookup the resolver IP address") + +def inputProcessor(self, filename): +""" +This is a specialised inputProcessor that also supports taking as +input a csv file. +""" +def csv_generator(fh): +for row in csv.reader(fh): +yield row[0] + +def simple_file_generator(fh): +for line in fh: +l = line.strip() +# Skip empty lines +if not l: +continue +# Skip comment lines +elif l.startswith('#'): +continue +yield l + +try: +fh = open(filename) +line = fh.readline() +# Detect the line of the citizenlab input file +if line.startswith("url,"): +generator = csv_generator(fh) +else: +fh.seek(0) +generator = simple_file_generator(fh) +for i in generator: +if not i.startswith("http"): +i = "http://{}/".format(i) +yield i +finally: +fh.close() + def setUp(self): """ Check for inputs. @@ -143,10 +181,11 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): } def experiment_dns_query(self): -log.msg("Doing DNS query for {}".format(self.hostname)) +log.msg("* doing DNS query for {}".format(self.hostname)) return self.performALookup(self.hostname) def tcp_connect(self, socket): +log.msg("* connecting to {}".format(socket)) ip_address, port = socket.split(":") port = int(port) result = { @@ -212,7 +251,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): try: control_headers_lower[header_name.lower()] except KeyError: -log.msg("Did not find the key {}".format(header_name)) +log.debug("Did not find the key {}".format(header_name)) return False count += 1 @@ -262,6 +301,10 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): control_asns = set(map(lambda x: geoip.IPToLocation(x)['asn'], control_ips)) +# Remove the instance of AS0 when we fail to find the ASN +control_asns.discard('AS0') +experiment_asns.discard('AS0') + if len(control_asns.intersection(experiment_asns)) > 0: return True @@ -316,11 +359,21 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['body_length_match'] == False)): blocking = 'dns' +# This happens when the DNS resolution is injected, but the domain +# doesn't have a valid record anymore or it resolves to an address +# that is only accessible from within the country/network of the probe. +elif (dns_consistent == False and + (self.control['dns']['failure'] is not None or + self.control['http_request']['failure'] is not None)): +blocking = 'dns' + return blocking @defer.inlineCallbacks def test_web_connectivity(self): +log.msg("") +log.msg("Starting test for {}".format(self.input)) experiment_dns = self.experiment_dns_query() @experiment_dns.addErrback @@ -385,15 +438,20 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): else: log.msg("* Is NOT accessible") self.report['accessible'] = False +log.msg("") def postProcessor(self, measurements): self.summary['accessible'] = self.summary.get('accessible', []) self.summary['not-accessible'] = self.summary.get('not-accessible', []) -self.summary['blocked'] = self.summary.get('blocked', []) +self.summary['blocked'] = self.summary.get('blocked', {}) if self.report['blocking'] not in (False, None): -self.summary['blocked'].append((self.input, -self.report['blocking'])) +
[tor-commits] [ooni-probe/master] Remove dead code
commit 29bbe9bd0b06b674267f0e7af4d1d914cb6fb79d Author: Arturo FilastòDate: Sun May 8 19:09:39 2016 +0200 Remove dead code --- ooni/templates/httpt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index c58daf2..a4844c8 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -113,7 +113,6 @@ class HTTPTest(NetTestCase): config.tor.socks_port)) self.report['socksproxy'] = None -sockshost, socksport = (None, None) if self.localOptions['socksproxy']: try: sockshost, socksport = self.localOptions['socksproxy'].split(':') ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Make the geoip lookup code more robust
commit 2627cd5ad2058d616cc037cd590b9f64112882a9 Author: Arturo FilastòDate: Mon May 2 17:54:36 2016 +0200 Make the geoip lookup code more robust --- ooni/geoip.py | 25 +++-- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/ooni/geoip.py b/ooni/geoip.py index dcc799e..fa6d1ae 100644 --- a/ooni/geoip.py +++ b/ooni/geoip.py @@ -34,25 +34,22 @@ def IPToLocation(ipaddr): asn_file = config.get_data_file_path('GeoIP/GeoIPASNum.dat') location = {'city': None, 'countrycode': 'ZZ', 'asn': 'AS0'} - -def error(): +if not asn_file or not country_file: log.err("Could not find GeoIP data file in data directories." "Try running ooniresources or" " edit your ooniprobe.conf") +return location -try: -country_dat = GeoIP(country_file) -location['countrycode'] = country_dat.country_code_by_addr(ipaddr) -if not location['countrycode']: -location['countrycode'] = 'ZZ' -except IOError: -error() +country_dat = GeoIP(country_file) +asn_dat = GeoIP(asn_file) -try: -asn_dat = GeoIP(asn_file) -location['asn'] = asn_dat.org_by_addr(ipaddr).split(' ')[0] -except: -error() +country_code = country_dat.country_code_by_addr(ipaddr) +if country_code is not None: +location['countrycode'] = country_code + +asn = asn_dat.org_by_addr(ipaddr) +if asn is not None: +location['asn'] = asn.split(' ')[0] return location ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Add support for computing header difference and follow redirects
commit 8c5fdd48ebb760b72b1b02a83fc96a19f92293c7 Author: Arturo FilastòDate: Sat Apr 16 14:30:29 2016 +0200 Add support for computing header difference and follow redirects --- ooni/nettests/blocking/web_connectivity.py | 54 +- 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 175c6d3..4b9b6c2 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -81,6 +81,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): } requiresRoot = False requiresTor = False +followRedirects = True # Factor used to determine HTTP blockpage detection factor = 0.8 @@ -112,6 +113,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['client_resolver'] = self.resolverIp self.report['dns_consistency'] = None self.report['body_length_match'] = None +self.report['headers_match'] = None + self.report['accessible'] = None self.report['blocking'] = None @@ -139,6 +142,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): } def experiment_dns_query(self): +log.msg("Doing DNS query for {}".format(self.hostname)) return self.performALookup(self.hostname) def tcp_connect(self, socket): @@ -197,6 +201,22 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): def experiment_http_get_request(self): return self.doRequest(self.input, headers=REQUEST_HEADERS) +def compare_headers(self, experiment_http_response): +count = 0 +control_headers_lower = {k.lower(): v for k, v in +self.report['control']['http_request']['headers'].items()} + +for header_name, header_value in \ +experiment_http_response.headers.getAllRawHeaders(): +try: +control_headers_lower[header_name.lower()] +except KeyError: +log.msg("Did not find the key {}".format(header_name)) +return False +count += 1 + +return count == len(self.report['control']['http_request']['headers']) + def compare_body_lengths(self, experiment_http_response): control_body_length = self.control['http_request']['body_length'] experiment_body_length = len(experiment_http_response.body) @@ -213,10 +233,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['body_proportion'] = rel if rel > float(self.factor): -self.report['body_length_match'] = True return True else: -self.report['body_length_match'] = False return False def compare_dns_experiments(self, experiment_dns_answers): @@ -229,16 +247,13 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): experiment_ips = set(experiment_dns_answers) if control_ips == experiment_ips: -self.report['dns_consistency'] = 'consistent' return True for experiment_ip in experiment_ips: if is_public_ipv4_address(experiment_ip) is False: -self.report['dns_consistency'] = 'inconsistent' return False if len(control_ips.intersection(experiment_ips)) > 0: -self.report['dns_consistency'] = 'consistent' return True experiment_asns = set(map(lambda x: geoip.IPToLocation(x)['asn'], @@ -247,10 +262,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): control_ips)) if len(control_asns.intersection(experiment_asns)) > 0: -self.report['dns_consistency'] = 'consistent' return True -self.report['dns_consistency'] = 'inconsistent' return False def compare_tcp_experiments(self): @@ -268,22 +281,26 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): def determine_blocking(self, experiment_http_response, experiment_dns_answers): blocking = False -body_length_match = None -dns_consistent = None -tcp_connect = None if (self.report['http_experiment_failure'] is None and self.report['control']['http_request']['failure'] is None): -body_length_match = self.compare_body_lengths(experiment_http_response) +self.report['body_length_match'] = self.compare_body_lengths( +experiment_http_response) +self.report['headers_match'] = self.compare_headers( +experiment_http_response) dns_consistent = self.compare_dns_experiments(experiment_dns_answers) +if dns_consistent is True: +self.report['dns_consistency'] = 'consistent' +else: +self.report['dns_consistency'] = 'inconsistent'
[tor-commits] [ooni-probe/master] Many improvements to web_connectivity test
commit 7e1ee49e44480f62ac93a9f63aa90a64342df2ed Author: Arturo FilastòDate: Thu Apr 14 22:54:49 2016 +0200 Many improvements to web_connectivity test * Run the probe resolver detection inside of the setupClass * Display a summary at the end of a test run summarising the results * Make the blocking detection logic more robust --- ooni/nettests/blocking/web_connectivity.py | 152 - 1 file changed, 108 insertions(+), 44 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index d8276b3..320f614 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -8,6 +8,7 @@ from ipaddr import IPv4Address, AddressValueError from twisted.internet import reactor from twisted.internet.protocol import Factory, Protocol from twisted.internet.endpoints import TCP4ClientEndpoint +from twisted.names import client, dns from twisted.internet import defer from twisted.python import usage @@ -19,6 +20,17 @@ from ooni.utils.net import StringProducer, BodyReceiver from ooni.templates import httpt, dnst from ooni.errors import failureToString +REQUEST_HEADERS = { +'User-Agent': ['Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, ' + 'like Gecko) Chrome/47.0.2526.106 Safari/537.36'], +'Accept-Language': ['en-US;q=0.8,en;q=0.5'], +'Accept': ['text/html,application/xhtml+xml,application/xml;q=0.9,' + '*/*;q=0.8'] +} + +class InvalidControlResponse(Exception): +pass + class TCPConnectProtocol(Protocol): def connectionMade(self): self.transport.loseConnection() @@ -72,6 +84,21 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): # Factor used to determine HTTP blockpage detection factor = 0.8 +resolverIp = None + +@classmethod +@defer.inlineCallbacks +def setUpClass(cls): +try: +answers = yield client.lookupAddress( +cls.localOptions['dns-discovery'] +) +assert len(answers) > 0 +assert len(answers[0]) > 0 +cls.resolverIp = answers[0][0].payload.dottedQuad() +except Exception as exc: +log.exception(exc) +log.err("Failed to lookup the resolver IP address") def setUp(self): """ @@ -82,7 +109,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): if not self.input: raise Exception("No input specified") -self.report['client_resolver'] = None +self.report['client_resolver'] = self.resolverIp self.report['dns_consistency'] = None self.report['body_length_match'] = None self.report['accessible'] = None @@ -111,9 +138,6 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): } } -def dns_discovery(self): -return self.performALookup(self.localOptions['dns-discovery']) - def experiment_dns_query(self): return self.performALookup(self.hostname) @@ -161,11 +185,17 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): finished = defer.Deferred() response.deliverBody(BodyReceiver(finished, content_length)) body = yield finished -self.control = json.loads(body) +try: +self.control = json.loads(body) +assert 'http_request' in self.control.keys() +assert 'tcp_connect' in self.control.keys() +assert 'dns' in self.control.keys() +except AssertionError, ValueError: +raise InvalidControlResponse(body) self.report['control'] = self.control def experiment_http_get_request(self): -return self.doRequest(self.input) +return self.doRequest(self.input, headers=REQUEST_HEADERS) def compare_body_lengths(self, experiment_http_response): control_body_length = self.control['http_request']['body_length'] @@ -233,27 +263,23 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): return success def determine_blocking(self, experiment_http_response, experiment_dns_answers): -blocking = None +blocking = False body_length_match = None dns_consistent = None tcp_connect = None -if self.report['control_failure'] is None and \ -self.report['http_experiment_failure'] is None and \ -self.report['control']['http_request']['failure'] is None: +if (self.report['http_experiment_failure'] is None and +self.report['control']['http_request']['failure'] is None): body_length_match = self.compare_body_lengths(experiment_http_response) -if self.report['control_failure'] is None: -dns_consistent = self.compare_dns_experiments(experiment_dns_answers) - -if self.report['control_failure'] is None: -
[tor-commits] [ooni-probe/master] Make the blocking detection heuristics more strict
commit ff77593d59dc7e25e4214e15571d616d7e374dce Author: Arturo FilastòDate: Mon Apr 18 18:45:09 2016 +0200 Make the blocking detection heuristics more strict --- ooni/nettests/blocking/web_connectivity.py | 16 +--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 4b9b6c2..28ee942 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -84,7 +84,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): followRedirects = True # Factor used to determine HTTP blockpage detection -factor = 0.8 +# the factor 0.7 comes from http://www3.cs.stonybrook.edu/~phillipa/papers/JLFG14.pdf +factor = 0.7 resolverIp = None @classmethod @@ -296,14 +297,23 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['dns_consistency'] = 'inconsistent' tcp_connect = self.compare_tcp_experiments() -if dns_consistent == True and tcp_connect == False: +if (dns_consistent == True and tcp_connect == False and +self.report['http_experiment_failure'] is not None): blocking = 'tcp_ip' +# XXX we may want to have different codes for these two types of +# blocking elif (dns_consistent == True and tcp_connect == True and self.report['body_length_match'] == False): blocking = 'http' +elif (dns_consistent == True and tcp_connect == True and +self.report['http_experiment_failure'] is not None and +self.report['control']['http_request']['failure'] != self.report['http_experiment_failure']): +blocking = 'http' -elif dns_consistent == False: +elif (dns_consistent == False and + (self.report['http_experiment_failure'] is not None or + self.report['body_length_match'] == False)): blocking = 'dns' return blocking ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] [feature] Add setUpClass method to NetTestCase
commit 9367f7f0398cfd0e2105596719c47009a5e65924 Author: Arturo FilastòDate: Thu Apr 14 22:54:25 2016 +0200 [feature] Add setUpClass method to NetTestCase --- ooni/director.py | 2 +- ooni/nettest.py| 18 +- ooni/tests/test_nettest.py | 6 -- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index 64b5a98..43ca36c 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -257,7 +257,7 @@ class Director(object): net_test = NetTest(test_cases, test_details, report) net_test.director = self -yield net_test.initializeInputProcessor() +yield net_test.initialize() try: self.activeNetTests.append(net_test) self.measurementManager.schedule(net_test.generateMeasurements()) diff --git a/ooni/nettest.py b/ooni/nettest.py index d01b3e4..67d3e27 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -566,14 +566,20 @@ class NetTest(object): return measurement @defer.inlineCallbacks -def initializeInputProcessor(self): +def initialize(self): for test_class, _ in self.testCases: +# Initialize Input Processor test_class.inputs = yield defer.maybeDeferred( test_class().getInputProcessor ) if not test_class.inputs: test_class.inputs = [None] +# Run the setupClass method +yield defer.maybeDeferred( +test_class.setUpClass +) + def generateMeasurements(self): """ This is a generator that yields measurements and registers the @@ -701,6 +707,16 @@ class NetTestCase(object): localOptions = {} +@classmethod +def setUpClass(cls): +""" +You can override this hook with logic that should be run once before +any test method in the NetTestCase is run. +This can be useful to populate class attribute that should be valid +for all the runtime of the NetTest. +""" +pass + def _setUp(self): """ This is the internal setup method to be overwritten by templates. diff --git a/ooni/tests/test_nettest.py b/ooni/tests/test_nettest.py index 91e663f..660fd3e 100644 --- a/ooni/tests/test_nettest.py +++ b/ooni/tests/test_nettest.py @@ -253,12 +253,13 @@ class TestNetTest(unittest.TestCase): ntl.loadNetTestString(net_test_string_with_required_option) self.assertRaises(MissingRequiredOption, ntl.checkOptions) +@defer.inlineCallbacks def test_net_test_inputs(self): ntl = NetTestLoader(dummyArgsWithFile) ntl.loadNetTestString(net_test_string_with_file) ntl.checkOptions() nt = NetTest(ntl.getTestCases(), ntl.getTestDetails(), None) -nt.initializeInputProcessor() +yield nt.initialize() # XXX: if you use the same test_class twice you will have consumed all # of its inputs! @@ -275,6 +276,7 @@ class TestNetTest(unittest.TestCase): ntl.checkOptions() self.assertEqual(dict(ntl.localOptions), dummyOptions) +@defer.inlineCallbacks def test_generate_measurements_size(self): ntl = NetTestLoader(dummyArgsWithFile) ntl.loadNetTestString(net_test_string_with_file) @@ -282,7 +284,7 @@ class TestNetTest(unittest.TestCase): net_test = NetTest(ntl.getTestCases(), ntl.getTestDetails(), None) -net_test.initializeInputProcessor() +yield net_test.initialize() measurements = list(net_test.generateMeasurements()) self.assertEqual(len(measurements), 20) ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Implement unittest to check if tests that fail twice
commit cf9ea6cb6f1cbd5455651291c8fe2c6822c804eb Author: Arturo FilastòDate: Tue May 3 12:27:54 2016 +0200 Implement unittest to check if tests that fail twice This is to ensure that net tests that eventually succeed, but fail sometimes will run properly until the end. --- ooni/tests/test_director.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ooni/tests/test_director.py b/ooni/tests/test_director.py index 5875ccb..d2c4c82 100644 --- a/ooni/tests/test_director.py +++ b/ooni/tests/test_director.py @@ -5,6 +5,8 @@ from ooni.director import Director from ooni.nettest import NetTestLoader from ooni.tests.bases import ConfigTestCase +from ooni.nettest import NetTestLoader + from twisted.internet import defer from twisted.trial import unittest ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Also record all the redirects
commit 8a745093ecd27240053b7e251de46eb4c303df6f Author: Arturo FilastòDate: Mon Apr 18 18:45:29 2016 +0200 Also record all the redirects * Add monkey patch for bug in twisted RedirectAgent: https://twistedmatrix.com/trac/ticket/8265 --- ooni/templates/httpt.py | 22 ++ ooni/utils/trueheaders.py | 42 +- 2 files changed, 55 insertions(+), 9 deletions(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 2b280f0..2a17f5b 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -7,6 +7,7 @@ from twisted.web.client import readBody, PartialDownloadError from twisted.internet import reactor from twisted.internet.endpoints import TCP4ClientEndpoint from ooni.utils.trueheaders import TrueHeadersAgent, TrueHeadersSOCKS5Agent +from ooni.utils.trueheaders import FixedRedirectAgent from ooni.nettest import NetTestCase from ooni.utils import log, base64Dict @@ -37,8 +38,15 @@ class StreamListener(StreamListenerMixin): log.err("Tor Exit ip detection failed") +def _representHeaders(headers): +represented_headers = {} +for name, value in headers.getAllRawHeaders(): +represented_headers[name] = value[0] +return represented_headers def _representBody(body): +if not body: +return body # XXX perhaps add support for decoding gzip in the future. body = body.replace('\0', '') decoded = False @@ -115,9 +123,8 @@ class HTTPTest(NetTestCase): if self.followRedirects: try: -from twisted.web.client import RedirectAgent -self.control_agent = RedirectAgent(self.control_agent) -self.agent = RedirectAgent(self.agent) +self.control_agent = FixedRedirectAgent(self.control_agent) +self.agent = FixedRedirectAgent(self.agent) self.report['agent'] = 'redirect' except: log.err("Warning! You are running an old version of twisted"\ @@ -147,11 +154,10 @@ class HTTPTest(NetTestCase): failure (instance): An instance of :class:twisted.internet.failure.Failure """ -def _representHeaders(headers): -represented_headers = {} -for name, value in headers.getAllRawHeaders(): -represented_headers[name] = value[0] -return represented_headers +if response and response.previousResponse: +self.addToReport(request, response.previousResponse, + response_body=None, + failure_string=None) log.debug("Adding %s to report" % request) request_headers = TrueHeaders(request['headers']) diff --git a/ooni/utils/trueheaders.py b/ooni/utils/trueheaders.py index d5aa206..a5b2fe3 100644 --- a/ooni/utils/trueheaders.py +++ b/ooni/utils/trueheaders.py @@ -7,9 +7,14 @@ import itertools from copy import copy -from twisted.web import client, _newclient, http_headers +from twisted.python.failure import Failure + +from twisted.web import client, _newclient, http_headers, error + from twisted.web._newclient import RequestNotSent, RequestGenerationFailed from twisted.web._newclient import TransportProxyProducer, STATUS +from twisted.web._newclient import ResponseFailed + from twisted.internet import reactor from twisted.internet.defer import Deferred, fail, maybeDeferred, failure @@ -179,3 +184,38 @@ class TrueHeadersSOCKS5Agent(SOCKS5Agent): self._wrappedAgent._pool = pool else: self._pool = pool + +class FixedRedirectAgent(client.BrowserLikeRedirectAgent): +""" +This is a redirect agent with this patch manually applied: +https://twistedmatrix.com/trac/ticket/8265 +""" +def _handleRedirect(self, response, method, uri, headers, redirectCount): +""" +Handle a redirect response, checking the number of redirects already +followed, and extracting the location header fields. + +This is pathed to fix a bug in infinite redirect loop. +""" +if redirectCount >= self._redirectLimit: +err = error.InfiniteRedirection( +response.code, +b'Infinite redirection detected', +location=uri) +raise ResponseFailed([Failure(err)], response) +locationHeaders = response.headers.getRawHeaders(b'location', []) +if not locationHeaders: +err = error.RedirectWithNoLocation( +response.code, b'No location header field', uri) +raise ResponseFailed([Failure(err)], response) +location = self._resolveLocation(response.request.absoluteURI, locationHeaders[0]) +deferred = self._agent.request(method, location, headers) + +def _chainResponse(newResponse): +newResponse.setPreviousResponse(response) +return
[tor-commits] [ooni-probe/master] Progress on implementing web_connectivity test
commit 9ab068b324290df91d4296f2f3388071df3bbac4 Author: Arturo FilastòDate: Tue Feb 2 16:07:04 2016 +0100 Progress on implementing web_connectivity test --- ooni/nettests/blocking/web_connectivity.py | 62 -- 1 file changed, 42 insertions(+), 20 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index bcc3a28..bc23f6d 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -13,7 +13,7 @@ from twisted.python import usage from ooni.utils import log from ooni.utils.net import StringProducer, BodyReceiver -from ooni.templates import httpt, dnst, tcpt +from ooni.templates import httpt, dnst from ooni.errors import failureToString class TCPConnectProtocol(Protocol): @@ -21,6 +21,7 @@ class TCPConnectProtocol(Protocol): self.transport.loseConnection() class TCPConnectFactory(Factory): +noisy = False def buildProtocol(self, addr): return TCPConnectProtocol() @@ -74,6 +75,9 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['accessible'] = None self.report['blocking'] = None +self.report['control_failure'] = None +self.report['experiment_failure'] = None + self.report['tcp_connect'] = [ ] @@ -83,8 +87,10 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.control = { 'tcp_connect': {}, -'dns_consistency': [], -'http_requests': { +'dns': { +'ips': [] +}, +'http_request': { 'body_length': None, 'headers': {} } @@ -93,7 +99,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): def dns_discovery(self): return self.performALookup(self.localOptions['dns-discovery']) -def dns_consistency(self): +def experiment_dns_query(self): return self.performALookup(self.hostname) def tcp_connect(self, socket): @@ -119,6 +125,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): @d.addErrback def eb(failure): result['status']['success'] = False +result['status']['failure'] = failureToString(failure) self.report['tcp_connect'].append(result) return d @@ -144,11 +151,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): def experiment_http_get_request(self): return self.doRequest(self.input) -def compare_control_experiment(self, experiment_http_response, - experiment_dns_answers): - -blocking = None - +def compare_body_lengths(self, experiment_http_response): control_body_length = self.control['http_request']['body_length'] experiment_body_length = len(experiment_http_response.body) @@ -165,20 +168,22 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['body_proportion'] = rel if rel > float(self.factor): self.report['body_length_match'] = True +return None else: -blocking = 'http' self.report['body_length_match'] = False +return 'http' +def compare_dns_experiments(self, experiment_dns_answers): control_ips = set(self.control['dns']['ips']) experiment_ips = set(experiment_dns_answers) if len(control_ips.intersection(experiment_ips)) > 0: self.report['dns_consistency'] = 'consistent' else: -if blocking is not None: -blocking = 'dns' self.report['dns_consistency'] = 'inconsistent' +def compare_tcp_experiments(self): +blocking = False for idx, result in enumerate(self.report['tcp_connect']): socket = "%s:%s" % (result['ip'], result['port']) control_status = self.control['tcp_connect'][socket] @@ -189,14 +194,13 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): blocking = 'tcp_ip' else: self.report['tcp_connect'][idx]['status']['blocked'] = False - -self.report['blocking'] = blocking +return blocking @defer.inlineCallbacks def test_web_connectivity(self): results = yield defer.DeferredList([ self.dns_discovery(), -self.dns_consistency() +self.experiment_dns_query() ]) self.report['client_resolver'] = None @@ -207,11 +211,29 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): sockets = map(lambda x: "%s:80" % x, results[1][1]) -dl = [self.control_request(sockets)] +control_request = self.control_request(sockets) +@control_request.addErrback +def control_err(failure): +self.report['control_failure'] =
[tor-commits] [ooni-probe/master] Add check on equality of control and experiment DNS responses
commit 6a4ed0161a961daaaccfd8b082a7328683f3b2fc Author: Arturo FilastòDate: Fri Apr 15 00:05:07 2016 +0200 Add check on equality of control and experiment DNS responses --- ooni/nettests/blocking/web_connectivity.py | 4 1 file changed, 4 insertions(+) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 320f614..175c6d3 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -228,6 +228,10 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): control_ips = set(self.control['dns']['ips']) experiment_ips = set(experiment_dns_answers) +if control_ips == experiment_ips: +self.report['dns_consistency'] = 'consistent' +return True + for experiment_ip in experiment_ips: if is_public_ipv4_address(experiment_ip) is False: self.report['dns_consistency'] = 'inconsistent' ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Maintain backward compatibility
commit d9656bc7187a10774bac3851b33fe4118b10c7a9 Author: Arturo FilastòDate: Sat Jan 30 12:36:49 2016 +0100 Maintain backward compatibility --- ooni/reporter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ooni/reporter.py b/ooni/reporter.py index 63d6f15..66a3e86 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -199,6 +199,7 @@ class YAMLReporter(OReporter): raise Exception("Failed to serialise entry") content += safe_dump(report_entry) content += '...\n' +report_entry.update(self.testDetails) self._write(content) def createReport(self): @@ -292,6 +293,7 @@ class OONIBReporter(OReporter): report_entry = entry else: raise Exception("Failed to serialise entry") +report_entry.update(self.testDetails) content += safe_dump(report_entry) content += '...\n' return content ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] Improve charset detection regexp
commit 55cd85351c02d7a7333f1c39e9993820b05b2b6d Author: Arturo FilastòDate: Sun May 8 19:08:22 2016 +0200 Improve charset detection regexp --- ooni/templates/httpt.py | 12 ++-- ooni/tests/test_templates.py | 3 +++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 51fba1a..6b4c4b9 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -1,4 +1,5 @@ import re +import codecs import random from txtorcon.interface import StreamListenerMixin @@ -17,7 +18,7 @@ from ooni.utils.net import StringProducer, userAgents from ooni.utils.trueheaders import TrueHeaders from ooni.errors import handleAllFailures -META_CHARSET_REGEXP = re.compile(']*?charset\s*=[\s"\']*([^\s"\'/>]+)') +META_CHARSET_REGEXP = re.compile(']*?charset\s*=[\s"\']*([^\s"\'/>!;]+)') class InvalidSocksProxyOption(Exception): pass @@ -56,7 +57,14 @@ def _representBody(body): # try to decode using that one first charset = META_CHARSET_REGEXP.search(body, re.IGNORECASE) if charset: -charsets.insert(0, charset.group(1)) +try: +encoding = charset.group(1).lower() +codecs.lookup(encoding) +charsets.insert(0, encoding) +except (LookupError, IndexError): +# Skip invalid codecs and partial regexp match +pass + for encoding in charsets: try: body = unicode(body, encoding) diff --git a/ooni/tests/test_templates.py b/ooni/tests/test_templates.py index e8fe636..ebd5b2e 100644 --- a/ooni/tests/test_templates.py +++ b/ooni/tests/test_templates.py @@ -55,8 +55,11 @@ class TestHTTPT(unittest.TestCase): """ with_charset_html = no_charset_html + '\n' with_empty_charset = no_charset_html + '\n' +with_two_charsets = no_charset_html + '\n' self.assertEqual(httpt.META_CHARSET_REGEXP.search(no_charset_html), None) self.assertEqual(httpt.META_CHARSET_REGEXP.search(with_charset_html).group(1), 'iso-8859-1') +self.assertEqual(httpt.META_CHARSET_REGEXP.search( +with_two_charsets).group(1), 'UTF-8') self.assertEqual(httpt.META_CHARSET_REGEXP.search(with_empty_charset), None) class TestDNST(unittest.TestCase): ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] First stab at implementing the web_connectivity test
commit a710467d95627c196f78f553e11055e16d9b9e7f Author: Arturo FilastòDate: Mon Feb 1 17:29:08 2016 +0100 First stab at implementing the web_connectivity test --- ooni/nettests/blocking/web_connectivity.py | 217 + 1 file changed, 217 insertions(+) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py new file mode 100644 index 000..bcc3a28 --- /dev/null +++ b/ooni/nettests/blocking/web_connectivity.py @@ -0,0 +1,217 @@ +# -*- encoding: utf-8 -*- + +import json +from urlparse import urlparse + +from twisted.internet import reactor +from twisted.internet.protocol import Factory, Protocol +from twisted.internet.endpoints import TCP4ClientEndpoint + +from twisted.internet import defer +from twisted.python import usage + +from ooni.utils import log + +from ooni.utils.net import StringProducer, BodyReceiver +from ooni.templates import httpt, dnst, tcpt +from ooni.errors import failureToString + +class TCPConnectProtocol(Protocol): +def connectionMade(self): +self.transport.loseConnection() + +class TCPConnectFactory(Factory): +def buildProtocol(self, addr): +return TCPConnectProtocol() + + +class UsageOptions(usage.Options): +optParameters = [ +['url', 'u', None, 'Specify a single URL to test'], +['dns-discovery', 'd', None, 'Specify the dns discovery test helper'], +['backend', 'b', None, 'The web_consistency backend test helper'], +] + + +class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): +""" +Web connectivity +""" +name = "Web connectivity" +description = ("Performs a HTTP GET request over Tor and one over the " + "local network and compares the two results.") +author = "Arturo Filastò" +version = "0.1.0" + +usageOptions = UsageOptions + +inputFile = [ +'file', 'f', None, 'List of URLS to perform GET requests to' +] + +requiredTestHelpers = { +'backend': 'web_connectivity', +'dns-discovery': 'dns_discovery' +} +requiresRoot = False +requiresTor = False + +# Factor used to determine HTTP blockpage detection +factor = 0.8 + +def setUp(self): +""" +Check for inputs. +""" +if self.localOptions['url']: +self.input = self.localOptions['url'] +if not self.input: +raise Exception("No input specified") + +self.report['client_resolver'] = None +self.report['dns_consistency'] = None +self.report['body_length_match'] = None +self.report['accessible'] = None +self.report['blocking'] = None + +self.report['tcp_connect'] = [ +] + +self.hostname = urlparse(self.input).netloc +if not self.hostname: +raise Exception("Invalid input") + +self.control = { +'tcp_connect': {}, +'dns_consistency': [], +'http_requests': { +'body_length': None, +'headers': {} +} +} + +def dns_discovery(self): +return self.performALookup(self.localOptions['dns-discovery']) + +def dns_consistency(self): +return self.performALookup(self.hostname) + +def tcp_connect(self, socket): +ip_address, port = socket.split(":") +port = int(port) +result = { +'ip': ip_address, +'port': port, +'status': { +'success': None, +'failure': None, +'blocked': None +} +} +point = TCP4ClientEndpoint(reactor, ip_address, port) +d = point.connect(TCPConnectFactory()) +@d.addCallback +def cb(p): +result['status']['success'] = True +result['status']['blocked'] = False +self.report['tcp_connect'].append(result) + +@d.addErrback +def eb(failure): +result['status']['success'] = False +self.report['tcp_connect'].append(result) +return d + +@defer.inlineCallbacks +def control_request(self, sockets): +bodyProducer = StringProducer(json.dumps({ +'http_request': self.input, +'tcp_connect': sockets +})) +response = yield self.agent.request("POST", +str(self.localOptions['backend']), +bodyProducer=bodyProducer) +try: +content_length = int(response.headers.getRawHeaders('content-length')[0]) +except Exception: +content_length = None + +finished = defer.Deferred() +response.deliverBody(BodyReceiver(finished, content_length)) +body = yield finished +self.control = json.loads(body) + +def experiment_http_get_request(self): +return self.doRequest(self.input) + +def
[tor-commits] [ooni-probe/master] Update the tests to be consistent with the new data formats
commit ff21910bcb2d2a3e54e5fc33be6b6d73ed0b60ee Author: Arturo FilastòDate: Thu Jan 28 13:09:36 2016 +0100 Update the tests to be consistent with the new data formats --- ooni/templates/httpt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index e608f8f..73bd5db 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -183,6 +183,7 @@ class HTTPTest(NetTestCase): session['failure'] = None if failure_string: session['failure'] = failure_string + self.report['requests'].append(session) def _processResponseBody(self, response_body, request, response, body_processor): ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [ooni-probe/master] [refactoring] Change signature of Deck to be more robust
commit aa8314973df0f6e458aa1e0de786faaad8264dc3 Author: Arturo FilastòDate: Fri Apr 15 15:45:03 2016 +0200 [refactoring] Change signature of Deck to be more robust Do more sanity checks on bouncer address at instantiation time --- ooni/deck.py | 30 ++ ooni/oonibclient.py | 15 +-- ooni/oonicli.py | 4 ++-- ooni/tests/mocks.py | 3 +++ ooni/tests/test_deck.py | 14 +++--- ooni/tests/test_templates.py | 3 +++ 6 files changed, 42 insertions(+), 27 deletions(-) diff --git a/ooni/deck.py b/ooni/deck.py index b13385d..4b2c502 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -95,28 +95,27 @@ def nettest_to_path(path, allow_arbitrary_paths=False): class Deck(InputFile): +# this exists so we can mock it out in unittests +_OONIBClient = OONIBClient + def __init__(self, deck_hash=None, - deckFile=None, + bouncer=None, decks_directory=config.decks_directory, no_collector=False): self.id = deck_hash -self.requiresTor = False self.no_collector = no_collector -self.bouncer = '' +self.bouncer = bouncer + +self.requiresTor = False + self.netTestLoaders = [] self.inputs = [] -self.oonibclient = OONIBClient(self.bouncer) - self.decksDirectory = os.path.abspath(decks_directory) -self.deckHash = deck_hash - -if deckFile: -self.loadDeck(deckFile) @property def cached_file(self): -return os.path.join(self.decksDirectory, self.deckHash) +return os.path.join(self.decksDirectory, self.id) @property def cached_descriptor(self): @@ -124,7 +123,7 @@ class Deck(InputFile): def loadDeck(self, deckFile): with open(deckFile) as f: -self.deckHash = sha256(f.read()).hexdigest() +self.id = sha256(f.read()).hexdigest() f.seek(0) test_deck = yaml.safe_load(f) @@ -175,7 +174,7 @@ class Deck(InputFile): @defer.inlineCallbacks def lookupCollectorAndTestHelpers(self): -self.oonibclient.address = self.bouncer +oonibclient = self._OONIBClient(self.bouncer) required_nettests = [] @@ -201,8 +200,7 @@ class Deck(InputFile): if not requires_test_helpers and not requires_collector: defer.returnValue(None) -log.debug("Looking up {}".format(required_nettests)) -response = yield self.oonibclient.lookupTestCollector(required_nettests) +response = yield oonibclient.lookupTestCollector(required_nettests) provided_net_tests = response['net-tests'] def find_collector_and_test_helpers(test_name, test_version, input_files): @@ -240,10 +238,10 @@ class Deck(InputFile): for i in net_test_loader.inputFiles: if i['url']: log.debug("Downloading %s" % i['url']) -self.oonibclient.address = i['address'] +oonibclient = self._OONIBClient(i['address']) try: -input_file = yield self.oonibclient.downloadInput(i['hash']) +input_file = yield oonibclient.downloadInput(i['hash']) except: raise e.UnableToLoadDeckInput diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py index 388bb68..336ae4e 100644 --- a/ooni/oonibclient.py +++ b/ooni/oonibclient.py @@ -1,5 +1,7 @@ import json +from urlparse import urljoin + from twisted.web.client import Agent from twisted.internet import defer, reactor from twisted.internet.endpoints import TCP4ClientEndpoint @@ -15,6 +17,15 @@ class OONIBClient(object): retries = 3 def __init__(self, address): +if address.startswith("https://;): +log.err("HTTPS bouncers are currently not supported!") +raise e.InvalidOONIBBouncerAddress +elif address.startswith("http://;): +log.msg("Warning using plaintext bouncer!") +elif address.startswith("httpo://"): +log.debug("Using Tor hidden service bouncer: {}".format(address)) +else: +raise e.InvalidOONIBBouncerAddress self.address = address def _request(self, method, urn, genReceiver, bodyProducer=None): @@ -30,7 +41,7 @@ class OONIBClient(object): raise e.InvalidOONIBBouncerAddress elif self.address.startswith('http://'): -log.msg("Warning using unencrypted backend") +log.msg("Warning using unencrypted bouncer") agent = Agent(reactor) attempts = 0 @@ -38,7 +49,7 @@ class OONIBClient(object): finished = defer.Deferred() def perform_request(attempts): -uri = address + urn +uri = urljoin(address, urn) d = agent.request(method, uri,
[tor-commits] [ooni-probe/master] Add detection for accessibility of the website
commit bcd87425f035a2f6d37278007a668cd5a0349bf8 Author: Arturo FilastòDate: Thu Feb 4 16:26:16 2016 +0100 Add detection for accessibility of the website * Minor refactoring of the test --- ooni/nettests/blocking/web_connectivity.py | 91 -- 1 file changed, 60 insertions(+), 31 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 8d83ad9..a61819a 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -89,7 +89,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['blocking'] = None self.report['control_failure'] = None -self.report['experiment_failure'] = None +self.report['http_experiment_failure'] = None +self.report['dns_experiment_failure'] = None self.report['tcp_connect'] = [] self.report['control'] = {} @@ -188,6 +189,11 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): return False def compare_dns_experiments(self, experiment_dns_answers): +if self.control['dns']['failure'] is not None and \ +self.control['dns']['failure'] == self.report['dns_experiment_failure']: +self.report['dns_consistency'] = 'consistent' +return True + control_ips = set(self.control['dns']['ips']) experiment_ips = set(experiment_dns_answers) @@ -225,11 +231,47 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['tcp_connect'][idx]['status']['blocked'] = False return success +def determine_blocking(self, experiment_http_response, experiment_dns_answers): +blocking = None +body_length_match = None +dns_consistent = None +tcp_connect = None + +if self.report['control_failure'] is None and \ +self.report['http_experiment_failure'] is None: +body_length_match = self.compare_body_lengths(experiment_http_response) + +if self.report['control_failure'] is None: +dns_consistent = self.compare_dns_experiments(experiment_dns_answers) + +if self.report['control_failure'] is None: +tcp_connect = self.compare_tcp_experiments() + +if dns_consistent == True and tcp_connect == False: +blocking = 'tcp_ip' + +elif dns_consistent == True and \ +tcp_connect == True and body_length_match == False: +blocking = 'http' + +elif dns_consistent == False: +blocking = 'dns' + +return blocking + + @defer.inlineCallbacks def test_web_connectivity(self): +experiment_dns = self.experiment_dns_query() + +@experiment_dns.addErrback +def dns_experiment_err(failure): +self.report['dns_experiment_failure'] = failureToString(failure) +return [] + results = yield defer.DeferredList([ self.dns_discovery(), -self.experiment_dns_query() +experiment_dns ]) self.report['client_resolver'] = None @@ -254,38 +296,25 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): experiment_http = self.experiment_http_get_request() @experiment_http.addErrback -def experiment_err(failure): -self.report['experiment_failure'] = failureToString(failure) +def http_experiment_err(failure): +self.report['http_experiment_failure'] = failureToString(failure) experiment_http_response = yield experiment_http -blocking = None -body_length_match = None -dns_consistent = None -tcp_connect = None - -if self.report['control_failure'] is None and \ -self.report['experiment_failure'] is None: -body_length_match = self.compare_body_lengths(experiment_http_response) - -if self.report['control_failure'] is None: -dns_consistent = self.compare_dns_experiments(experiment_dns_answers) - -if self.report['control_failure'] is None: -tcp_connect = self.compare_tcp_experiments() - -if dns_consistent == True and tcp_connect == False: -blocking = 'tcp_ip' - -elif dns_consistent == True and \ -tcp_connect == True and body_length_match == False: -blocking = 'http' - -elif dns_consistent == False: -blocking = 'dns' - +blocking = self.determine_blocking(experiment_http_response, experiment_dns_answers) self.report['blocking'] = blocking + if blocking is not None: -log.msg("Blocking detected on %s due to %s" % (self.input, blocking)) +log.msg("%s: BLOCKING DETECTED due to %s" % (self.input, blocking)) +else: +log.msg("%s: No blocking detected" % self.input) +
[tor-commits] [ooni-probe/master] Clean up how we handle the reportId and remove unused functions
commit da51f6d069ac55ebf430ebb0adfa6a70110fda68 Author: Arturo FilastòDate: Tue Apr 12 17:09:51 2016 +0200 Clean up how we handle the reportId and remove unused functions --- ooni/deck.py| 52 --- ooni/director.py| 8 +- ooni/reporter.py| 66 +++-- ooni/tasks.py | 3 --- ooni/tests/mocks.py | 19 + ooni/tests/test_deck.py | 4 +-- ooni/tests/test_reporter.py | 2 +- 7 files changed, 55 insertions(+), 99 deletions(-) diff --git a/ooni/deck.py b/ooni/deck.py index 4b6d894..b13385d 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -171,10 +171,10 @@ class Deck(InputFile): if self.bouncer: log.msg("Looking up collector and test helpers") -yield self.lookupCollector() +yield self.lookupCollectorAndTestHelpers() @defer.inlineCallbacks -def lookupCollector(self): +def lookupCollectorAndTestHelpers(self): self.oonibclient.address = self.bouncer required_nettests = [] @@ -221,9 +221,9 @@ class Deck(InputFile): net_test_loader.testName) collector, test_helpers = \ -find_collector_and_test_helpers(net_test_loader.testName, -net_test_loader.testVersion, -net_test_loader.inputFiles) + find_collector_and_test_helpers(test_name=net_test_loader.testName, + test_version=net_test_loader.testVersion, + input_files=net_test_loader.inputFiles) for option, name in net_test_loader.missingTestHelpers: test_helper_address = test_helpers[name].encode('utf-8') @@ -234,48 +234,6 @@ class Deck(InputFile): net_test_loader.collector = collector.encode('utf-8') @defer.inlineCallbacks -def lookupTestHelpers(self): -self.oonibclient.address = self.bouncer - -required_test_helpers = [] -requires_collector = [] -for net_test_loader in self.netTestLoaders: -if not net_test_loader.collector and not self.no_collector: -requires_collector.append(net_test_loader) - -required_test_helpers += map(lambda x: x[1], - net_test_loader.missingTestHelpers) - -if not required_test_helpers and not requires_collector: -defer.returnValue(None) - -response = yield self.oonibclient.lookupTestHelpers(required_test_helpers) - -for net_test_loader in self.netTestLoaders: -log.msg("Setting collector and test helpers for %s" % -net_test_loader.testName) - -# Only set the collector if the no collector has been specified -# from the command line or via the test deck. -if len(net_test_loader.missingTestHelpers) == 0 and \ -net_test_loader in requires_collector: -log.msg("Using the default collector: %s" % -response['default']['collector']) -net_test_loader.collector = response['default']['collector'].encode('utf-8') -continue - -for option, name in net_test_loader.missingTestHelpers: -test_helper_address = response[name]['address'].encode('utf-8') -test_helper_collector = \ -response[name]['collector'].encode('utf-8') - -log.msg("Using this helper: %s" % test_helper_address) -net_test_loader.localOptions[option] = test_helper_address -net_test_loader.testHelpers[option] = test_helper_address -if net_test_loader in requires_collector: -net_test_loader.collector = test_helper_collector - -@defer.inlineCallbacks def fetchAndVerifyNetTestInput(self, net_test_loader): """ fetch and verify a single NetTest's inputs """ log.debug("Fetching and verifying inputs") diff --git a/ooni/director.py b/ooni/director.py index 13a4503..64b5a98 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -253,16 +253,10 @@ class Director(object): self.reportEntryManager, collector_address, no_yamloo) +yield report.open() net_test = NetTest(test_cases, test_details, report) net_test.director = self -yield net_test.report.open() - -# XXX this needs some serious refactoring -net_test_loader.reportID = report.reportID -net_test.reportID = report.reportID -net_test.testDetails['report_id'] = report.reportID - yield net_test.initializeInputProcessor() try:
[tor-commits] [ooni-probe/master] Implement basic web connectivity test
commit d92a95b628167043a84ceabac77e45f3cd319d3d Author: Arturo FilastòDate: Wed Feb 3 19:36:11 2016 +0100 Implement basic web connectivity test * Various hacky fixes to enforce correct report format and handling of report_id --- ooni/director.py | 5 ++ ooni/nettest.py| 6 ++- ooni/nettests/blocking/web_connectivity.py | 86 -- ooni/reporter.py | 7 ++- ooni/tasks.py | 3 ++ ooni/templates/httpt.py| 19 +++ 6 files changed, 94 insertions(+), 32 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index 02619c2..13a4503 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -258,6 +258,11 @@ class Director(object): yield net_test.report.open() +# XXX this needs some serious refactoring +net_test_loader.reportID = report.reportID +net_test.reportID = report.reportID +net_test.testDetails['report_id'] = report.reportID + yield net_test.initializeInputProcessor() try: self.activeNetTests.append(net_test) diff --git a/ooni/nettest.py b/ooni/nettest.py index 074e8c2..166451a 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -161,6 +161,8 @@ class NetTestLoader(object): method_prefix = 'test' collector = None yamloo = True +requiresTor = False +reportID = None def __init__(self, options, test_file=None, test_string=None, annotations={}): @@ -568,10 +570,12 @@ class NetTest(object): @defer.inlineCallbacks def initializeInputProcessor(self): -for test_class, test_method in self.testCases: +for test_class, _ in self.testCases: test_class.inputs = yield defer.maybeDeferred( test_class().getInputProcessor ) +if not test_class.inputs: +test_class.inputs = [None] def generateMeasurements(self): """ diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index bc23f6d..8d83ad9 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -3,6 +3,8 @@ import json from urlparse import urlparse +from ipaddr import IPv4Address, AddressValueError + from twisted.internet import reactor from twisted.internet.protocol import Factory, Protocol from twisted.internet.endpoints import TCP4ClientEndpoint @@ -10,6 +12,7 @@ from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.internet import defer from twisted.python import usage +from ooni import geoip from ooni.utils import log from ooni.utils.net import StringProducer, BodyReceiver @@ -34,6 +37,16 @@ class UsageOptions(usage.Options): ] +def is_public_ipv4_address(address): +try: +ip_address = IPv4Address(address) +if not any([ip_address.is_private, +ip_address.is_loopback]): +return True +return False +except AddressValueError: +return None + class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): """ Web connectivity @@ -51,8 +64,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): ] requiredTestHelpers = { -'backend': 'web_connectivity', -'dns-discovery': 'dns_discovery' +'backend': 'web-connectivity', +'dns-discovery': 'dns-discovery' } requiresRoot = False requiresTor = False @@ -78,8 +91,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['control_failure'] = None self.report['experiment_failure'] = None -self.report['tcp_connect'] = [ -] +self.report['tcp_connect'] = [] +self.report['control'] = {} self.hostname = urlparse(self.input).netloc if not self.hostname: @@ -147,6 +160,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): response.deliverBody(BodyReceiver(finished, content_length)) body = yield finished self.control = json.loads(body) +self.report['control'] = self.control def experiment_http_get_request(self): return self.doRequest(self.input) @@ -168,33 +182,48 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['body_proportion'] = rel if rel > float(self.factor): self.report['body_length_match'] = True -return None +return True else: self.report['body_length_match'] = False -return 'http' +return False def compare_dns_experiments(self, experiment_dns_answers): control_ips = set(self.control['dns']['ips']) experiment_ips = set(experiment_dns_answers) +for experiment_ip in experiment_ips: +if
[tor-commits] [ooni-probe/master] Fix report uploader to work with JSON
commit 613a02cdfe44c14fc8a1e91003a8617d913f Author: Arturo FilastòDate: Thu Feb 4 16:39:14 2016 +0100 Fix report uploader to work with JSON * Add better exception handling to deckgen * We should actually be using POST since PUT is indempotent * Debugging of backward and forward data format compatibility * Better integrity checks in web_consistency --- ooni/nettests/blocking/web_connectivity.py | 4 +++- ooni/reporter.py | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index a61819a..6734618 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -106,6 +106,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): }, 'http_request': { 'body_length': None, +'failure': True, 'headers': {} } } @@ -238,7 +239,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): tcp_connect = None if self.report['control_failure'] is None and \ -self.report['http_experiment_failure'] is None: +self.report['http_experiment_failure'] is None and \ +self.report['control']['http_request']['failure'] is None: body_length_match = self.compare_body_lengths(experiment_http_response) if self.report['control_failure'] is None: diff --git a/ooni/reporter.py b/ooni/reporter.py index 49df443..4318fd5 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -201,7 +201,6 @@ class YAMLReporter(OReporter): raise Exception("Failed to serialise entry") content += safe_dump(report_entry) content += '...\n' -report_entry.update(self.testDetails) self._write(content) def createReport(self): @@ -295,7 +294,6 @@ class OONIBReporter(OReporter): report_entry = entry else: raise Exception("Failed to serialise entry") -report_entry.update(self.testDetails) content += safe_dump(report_entry) content += '...\n' return content ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [translation/tails-openpgp-applet] Update translations for tails-openpgp-applet
commit c092e33521ca7ff1dab3ea7a35b89d212d87694b Author: Translation commit botDate: Mon May 30 14:17:05 2016 + Update translations for tails-openpgp-applet --- et/openpgp-applet.pot | 7 --- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/et/openpgp-applet.pot b/et/openpgp-applet.pot index b3178a6..32e50ca 100644 --- a/et/openpgp-applet.pot +++ b/et/openpgp-applet.pot @@ -3,13 +3,14 @@ # This file is distributed under the same license as the PACKAGE package. # # Translators: +# Kristjan Räts , 2016 msgid "" msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: ta...@boum.org\n" "POT-Creation-Date: 2015-08-10 15:55+0200\n" -"PO-Revision-Date: 2015-11-23 02:23+\n" -"Last-Translator: FULL NAME \n" +"PO-Revision-Date: 2016-05-30 13:49+\n" +"Last-Translator: Kristjan Räts \n" "Language-Team: Estonian (http://www.transifex.com/otf/torproject/language/et/)\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -43,7 +44,7 @@ msgstr "" #: bin/openpgp-applet:243 msgid "_Decrypt/Verify Clipboard" -msgstr "" +msgstr "_Dekrüpteeri/Kontrolli lõikepuhvrit" #: bin/openpgp-applet:247 msgid "_Manage Keys" ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [translation/torbutton-abouttorproperties_completed] Update translations for torbutton-abouttorproperties_completed
commit 825a612ee4c784e6dd652c9b60fff087baa569fe Author: Translation commit botDate: Mon May 30 14:16:28 2016 + Update translations for torbutton-abouttorproperties_completed --- et/abouttor.properties | 21 + 1 file changed, 21 insertions(+) diff --git a/et/abouttor.properties b/et/abouttor.properties new file mode 100644 index 000..dbbeb4a --- /dev/null +++ b/et/abouttor.properties @@ -0,0 +1,21 @@ +# Copyright (c) 2014, The Tor Project, Inc. +# See LICENSE for licensing information. +# vim: set sw=2 sts=2 ts=8 et: + +aboutTor.searchSP.privacy=Otsi turvaliselt avalehega. +# The following string is a link which replaces %1$S above. +aboutTor.searchSP.privacy.link=https://startpage.com/eng/protect-privacy.html +# The following string is a link which replaces %2$S above. +aboutTor.searchSP.search.link=https://startpage.com/ + +aboutTor.searchDDG.privacy=Otsi turvaliselt DickDuckGoga. +# The following string is a link which replaces %1$S above. +aboutTor.searchDDG.privacy.link=https://duckduckgo.com/privacy.html +# The following string is a link which replaces %2$S above. +aboutTor.searchDDG.search.link=https://duckduckgo.com/ + +aboutTor.searchDC.privacy=Otsi turvaliselt Disconnect.mega. +# The following string is a link which replaces %1$S above. +aboutTor.searchDC.privacy.link=https://disconnect.me/privacy +# The following string is a link which replaces %2$S above. +aboutTor.searchDC.search.link=https://search.disconnect.me/ ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [translation/torbutton-aboutdialogdtd] Update translations for torbutton-aboutdialogdtd
commit 66550db622de8575b07dfaf2d93e9801102d87c3 Author: Translation commit botDate: Mon May 30 14:16:18 2016 + Update translations for torbutton-aboutdialogdtd --- et/aboutdialog.dtd | 22 +++--- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/et/aboutdialog.dtd b/et/aboutdialog.dtd index 5099ad7..98fb292 100644 --- a/et/aboutdialog.dtd +++ b/et/aboutdialog.dtd @@ -1,19 +1,19 @@ - + - - + + - + - - + + - + - + - + - - + + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [translation/tails-perl5lib_completed] Update translations for tails-perl5lib_completed
commit e4dbcd36d52f3449339eb4e857feb0299466f198 Author: Translation commit botDate: Mon May 30 14:16:11 2016 + Update translations for tails-perl5lib_completed --- et.po | 35 +++ 1 file changed, 35 insertions(+) diff --git a/et.po b/et.po new file mode 100644 index 000..96a92c9 --- /dev/null +++ b/et.po @@ -0,0 +1,35 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR Tails developers +# This file is distributed under the same license as the PACKAGE package. +# +# Translators: +# Kristjan Räts , 2016 +msgid "" +msgstr "" +"Project-Id-Version: The Tor Project\n" +"Report-Msgid-Bugs-To: Tails developers \n" +"POT-Creation-Date: 2016-04-24 16:36+0200\n" +"PO-Revision-Date: 2016-05-30 14:13+\n" +"Last-Translator: Kristjan Räts \n" +"Language-Team: Estonian (http://www.transifex.com/otf/torproject/language/et/)\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Language: et\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" + +#: ../lib/Tails/RunningSystem.pm:159 +msgid "Error" +msgstr "Viga" + +#: ../lib/Tails/RunningSystem.pm:161 +msgid "" +"The device Tails is running from cannot be found. Maybe you used the `toram'" +" option?" +msgstr "Seade, millel Tails käivitati, ei ole leitav. Võib-olla kasutasid parameetrit 'toram'?" + +#: ../lib/Tails/RunningSystem.pm:192 +msgid "" +"The drive Tails is running from cannot be found. Maybe you used the `toram' " +"option?" +msgstr "Ketas, millel Tails käivitati, ei ole leitav. Võib-olla kasutasid parameetrit 'toram'?" ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [translation/torbutton-aboutdialogdtd_completed] Update translations for torbutton-aboutdialogdtd_completed
commit dda3e8bf32f849933e877b411973a80775245da4 Author: Translation commit botDate: Mon May 30 14:16:21 2016 + Update translations for torbutton-aboutdialogdtd_completed --- et/aboutdialog.dtd | 19 +++ 1 file changed, 19 insertions(+) diff --git a/et/aboutdialog.dtd b/et/aboutdialog.dtd new file mode 100644 index 000..98fb292 --- /dev/null +++ b/et/aboutdialog.dtd @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [translation/torbutton-abouttorproperties] Update translations for torbutton-abouttorproperties
commit 35bfc1bb48919403b084a6cb052d801d0238adef Author: Translation commit botDate: Mon May 30 14:16:25 2016 + Update translations for torbutton-abouttorproperties --- et/abouttor.properties | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/et/abouttor.properties b/et/abouttor.properties index d607324..dbbeb4a 100644 --- a/et/abouttor.properties +++ b/et/abouttor.properties @@ -2,19 +2,19 @@ # See LICENSE for licensing information. # vim: set sw=2 sts=2 ts=8 et: -aboutTor.searchSP.privacy=Search securely with Startpage. +aboutTor.searchSP.privacy=Otsi turvaliselt avalehega. # The following string is a link which replaces %1$S above. aboutTor.searchSP.privacy.link=https://startpage.com/eng/protect-privacy.html # The following string is a link which replaces %2$S above. aboutTor.searchSP.search.link=https://startpage.com/ -aboutTor.searchDDG.privacy=Search securely with DuckDuckGo. +aboutTor.searchDDG.privacy=Otsi turvaliselt DickDuckGoga. # The following string is a link which replaces %1$S above. aboutTor.searchDDG.privacy.link=https://duckduckgo.com/privacy.html # The following string is a link which replaces %2$S above. aboutTor.searchDDG.search.link=https://duckduckgo.com/ -aboutTor.searchDC.privacy=Search securely with Disconnect.me. +aboutTor.searchDC.privacy=Otsi turvaliselt Disconnect.mega. # The following string is a link which replaces %1$S above. aboutTor.searchDC.privacy.link=https://disconnect.me/privacy # The following string is a link which replaces %2$S above. ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [translation/tails-perl5lib] Update translations for tails-perl5lib
commit e870383958daecb8ace4f302515df779ea419d86 Author: Translation commit botDate: Mon May 30 14:16:07 2016 + Update translations for tails-perl5lib --- et.po | 13 +++-- 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/et.po b/et.po index 6d4835a..96a92c9 100644 --- a/et.po +++ b/et.po @@ -3,13 +3,14 @@ # This file is distributed under the same license as the PACKAGE package. # # Translators: +# Kristjan Räts , 2016 msgid "" msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: Tails developers \n" -"POT-Creation-Date: 2016-01-11 15:40+0100\n" -"PO-Revision-Date: 2016-01-12 08:13+\n" -"Last-Translator: carolyn \n" +"POT-Creation-Date: 2016-04-24 16:36+0200\n" +"PO-Revision-Date: 2016-05-30 14:13+\n" +"Last-Translator: Kristjan Räts \n" "Language-Team: Estonian (http://www.transifex.com/otf/torproject/language/et/)\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -19,16 +20,16 @@ msgstr "" #: ../lib/Tails/RunningSystem.pm:159 msgid "Error" -msgstr "" +msgstr "Viga" #: ../lib/Tails/RunningSystem.pm:161 msgid "" "The device Tails is running from cannot be found. Maybe you used the `toram'" " option?" -msgstr "" +msgstr "Seade, millel Tails käivitati, ei ole leitav. Võib-olla kasutasid parameetrit 'toram'?" #: ../lib/Tails/RunningSystem.pm:192 msgid "" "The drive Tails is running from cannot be found. Maybe you used the `toram' " "option?" -msgstr "" +msgstr "Ketas, millel Tails käivitati, ei ole leitav. Võib-olla kasutasid parameetrit 'toram'?" ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [translation/tails-misc] Update translations for tails-misc
commit 292091ee6da5bedc5c200952e9c30bc4709e7ee3 Author: Translation commit botDate: Mon May 30 14:15:51 2016 + Update translations for tails-misc --- et.po | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/et.po b/et.po index 273a531..e744b2e 100644 --- a/et.po +++ b/et.po @@ -8,7 +8,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2016-05-25 20:25+0200\n" -"PO-Revision-Date: 2016-05-30 13:36+\n" +"PO-Revision-Date: 2016-05-30 14:11+\n" "Last-Translator: carolyn \n" "Language-Team: Estonian (http://www.transifex.com/otf/torproject/language/et/)\n" "MIME-Version: 1.0\n" @@ -91,7 +91,7 @@ msgstr "" #: config/chroot_local-includes/usr/local/bin/gpgApplet:200 msgid "_Decrypt/Verify Clipboard" -msgstr "" +msgstr "_Dekrüpteeri/Kontrolli lõikepuhvrit" #: config/chroot_local-includes/usr/local/bin/gpgApplet:204 msgid "_Manage Keys" @@ -348,7 +348,7 @@ msgstr "" #: config/chroot_local-includes/usr/local/bin/tails-upgrade-frontend-wrapper:25 #: config/chroot_local-includes/usr/local/sbin/unsafe-browser:28 msgid "Error" -msgstr "" +msgstr "Viga" #: config/chroot_local-includes/usr/local/bin/tails-upgrade-frontend-wrapper:45 msgid "" ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [translation/tails-persistence-setup] Update translations for tails-persistence-setup
commit 37faefc09f83c54c7695cae255e33a34427f73ad Author: Translation commit botDate: Mon May 30 14:15:27 2016 + Update translations for tails-persistence-setup --- et/et.po | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/et/et.po b/et/et.po index d86b84e..abe4b30 100644 --- a/et/et.po +++ b/et/et.po @@ -9,7 +9,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: Tails developers \n" "POT-Creation-Date: 2016-04-24 16:40+0200\n" -"PO-Revision-Date: 2016-05-30 13:45+\n" +"PO-Revision-Date: 2016-05-30 14:11+\n" "Last-Translator: Kristjan Räts \n" "Language-Team: Estonian (http://www.transifex.com/otf/torproject/language/et/)\n" "MIME-Version: 1.0\n" @@ -137,7 +137,7 @@ msgstr "" #: ../lib/Tails/Persistence/Setup.pm:312 ../lib/Tails/Persistence/Setup.pm:459 msgid "Error" -msgstr "" +msgstr "Viga" #: ../lib/Tails/Persistence/Setup.pm:344 #, perl-format ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [translation/https_everywhere] Update translations for https_everywhere
commit f682ec3a3fce8d050397117e326f6842c5545e76 Author: Translation commit botDate: Mon May 30 14:15:15 2016 + Update translations for https_everywhere --- et/ssl-observatory.dtd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/et/ssl-observatory.dtd b/et/ssl-observatory.dtd index da75df6..dd033be 100644 --- a/et/ssl-observatory.dtd +++ b/et/ssl-observatory.dtd @@ -20,7 +20,7 @@ to turn it on?">--> ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [translation/https_everywhere_completed] Update translations for https_everywhere_completed
commit 922df1443011a7f284b28053b924ec7a1bfb3837 Author: Translation commit botDate: Mon May 30 14:15:21 2016 + Update translations for https_everywhere_completed --- et/ssl-observatory.dtd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/et/ssl-observatory.dtd b/et/ssl-observatory.dtd index da75df6..dd033be 100644 --- a/et/ssl-observatory.dtd +++ b/et/ssl-observatory.dtd @@ -20,7 +20,7 @@ to turn it on?">--> ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [translation/torbirdy_completed] Update translations for torbirdy_completed
commit 6d9b0dee869dfbec44e1b518ae81afb3c7912a2d Author: Translation commit botDate: Mon May 30 14:15:38 2016 + Update translations for torbirdy_completed --- et/torbirdy.properties | 18 ++ 1 file changed, 18 insertions(+) diff --git a/et/torbirdy.properties b/et/torbirdy.properties new file mode 100644 index 000..f91bb8f --- /dev/null +++ b/et/torbirdy.properties @@ -0,0 +1,18 @@ +torbirdy.name=TorBirdy + +torbirdy.enabled.tor=TorBirdy aktiivne: Tor +torbirdy.enabled.jondo=TorBirdy aktiivne: JonDo +torbirdy.enabled.custom=TorBirdy aktiivne: kohandatud proksi +torbirdy.enabled.torification=TorBirdy aktiivne: läbipaistev Tor'indamine +torbirdy.enabled.whonix=TorBirdy aktiivne: Whonix +torbirdy.disabled=TorBirdy: mitteaktiivne! +torbirdy.enabled=TorBirdy: aktiivne + +torbirdy.email.prompt=TorBirdy keelas Thunderbirdi automaathäälestuse visardi, et kaitsta su anonüümsust.\n\nSoovitatavad turvalisuse seaded on %S-le määratud.\n\nSa saad konto ülejäänud seaded häälestada käsitsi. + +torbirdy.email.advanced=Palun aevesta, et TorBirdy täpsemate seadete muutmine ei ole soovitatav.\n\nSa peaksid jätkama ainult siis, kui oled kindel selles, mida sa teed. +torbirdy.email.advanced.nextwarning=Kuva seda hoiatust ka järgmisel korral +torbirdy.email.advanced.title=TorBirdy täpsemad seaded + +torbirdy.firstrun=Sa kastad nüüd TorBirdy't.\n\nSinu anonüümsuse tagamiseks muudab ta Thunderbirdy sätted, mis ta määras, sinu või teiste laiendite poolt mittemuudetavateks. Mõningaid sätteid saab muuta ja selleks kasuta TorBirdy seadete dialoogi. Kui TorBirdy eemaldatakse või deaktiveeritakse, siis lähtestatakse need seaded nende TorBirdy paigaldamise eelsetele väärtustele.\n\nKui sa oled uus kasutaja, siis on soovitatav, et sa loeksid käbi TorBirdy veebilehe, et saada aru, mida me proovime oma kasutajatele TorBirdyga pakkuda. +torbirdy.website=https://trac.torproject.org/projects/tor/wiki/torbirdy ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [translation/tor-messenger-privdtd] Update translations for tor-messenger-privdtd
commit 37dfd6b3ce63caa24985c286abcacf3a6f7e0847 Author: Translation commit botDate: Mon May 30 13:46:53 2016 + Update translations for tor-messenger-privdtd --- et/priv.dtd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/et/priv.dtd b/et/priv.dtd index 2e65400..c62bb33 100644 --- a/et/priv.dtd +++ b/et/priv.dtd @@ -1,2 +1,2 @@ - + \ No newline at end of file ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [translation/torbutton-branddtd_completed] Update translations for torbutton-branddtd_completed
commit 5041bb375d174c41d3007ccdff897a73efc1e265 Author: Translation commit botDate: Mon May 30 13:46:02 2016 + Update translations for torbutton-branddtd_completed --- et/brand.dtd | 15 +++ 1 file changed, 15 insertions(+) diff --git a/et/brand.dtd b/et/brand.dtd new file mode 100644 index 000..a47c121 --- /dev/null +++ b/et/brand.dtd @@ -0,0 +1,15 @@ + + + + + + + + + + + + + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [translation/torbutton-torbuttonproperties] Update translations for torbutton-torbuttonproperties
commit b53ae43b31c9c8add62e1eae5348d0cfb0c572cd Author: Translation commit botDate: Mon May 30 13:46:05 2016 + Update translations for torbutton-torbuttonproperties --- et/torbutton.properties | 18 +- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/et/torbutton.properties b/et/torbutton.properties index 51e1568..0e728a3 100644 --- a/et/torbutton.properties +++ b/et/torbutton.properties @@ -42,17 +42,17 @@ torbutton.popup.captcha.ask = Tor tuvastas Google CAPTCHA. Kas sa soovid, et sin torbutton.popup.captcha.always = Nüüdsest edasi soorita alati see tegevus torbutton.popup.redirect = Suuna ümber torbutton.popup.no_redirect = Ãra suuna ümber -torbutton.popup.prompted_language = To give you more privacy, Torbutton can request the English language version of web pages. This may cause web pages that you prefer to read in your native language to display in English instead.\n\nWould you like to request English language web pages for better privacy? -torbutton.popup.no_newnym = Torbutton cannot safely give you a new identity. It does not have access to the Tor Control Port.\n\nAre you running Tor Browser Bundle? -torbutton.popup.use_tbb = Näib, et sa kasutat Torbuttonit koos Firefoxiga, mis pole enam soovitatud turvaline lahendus.\n\nMe soovitame sul selle asemel hankida Tori Veebilehitseja Pakett saates e-posti aadressile get...@torproject.org või laadides selle alla järgnevalt aadressilt: -torbutton.popup.pref_error = Torbutton cannot update preferences in the Tor Browser profile directory. -torbutton.popup.permission_denied = Please either reset the permissions of the Tor Browser directory or copy it to a new location. -torbutton.popup.device_full = The disk appears to be full. Please free up space or move Tor Browser to a new device. +torbutton.popup.prompted_language = Täiendava privaatsuse andmiseks võib Torbutton küsida veebilehe inglise keelset versiooni. Selle tulemusel võidakse sulle kuvada lehti, mida loed hetkel oma keeles hoopis inglise keeles.\n\nKas sa soovid parema privaatsuse nimel küsida inglise keelseid veebilehti? +torbutton.popup.no_newnym = Torbutton ei saa sulle turvaliselt anda uut identiteeti. Tal puudub pilipääs Tor'i juhtpordile.\n\nKas sa kasutad Tor Brauseri paketti? +torbutton.popup.use_tbb = Näib, et sa kasutat Torbuttonit koos Firefoxiga, mis pole enam soovitatud turvaline lahendus.\n\nMe soovitame sul selle asemel hankida Tor'i Browser paketi saates e-kirja aadressile get...@torproject.org või laadides selle alla järgnevalt aadressilt: +torbutton.popup.pref_error = Torbutton ei saa uuendada eelistusi Tor Brauseri profiili kataloogis. +torbutton.popup.permission_denied = Palun lähtesta Tor Brauseri kataloogi õigused või kopeeri kataloog uude asukohta. +torbutton.popup.device_full = Ketas tundub olevat täis. Palun tee rohkem vaba ruumi või liiguta Tor Brauser uuele seadmele. torbutton.title.prompt_torbrowser = Oluline Torbuttoni informatsioon -torbutton.popup.prompt_torbrowser = Torbutton works differently now: you can't turn it off any more.\n\nWe made this change because it isn't safe to use Torbutton in a browser that's also used for non-Tor browsing. There were too many bugs there that we couldn't fix any other way.\n\nIf you want to keep using Firefox normally, you should uninstall Torbutton and download Tor Browser Bundle. The privacy properties of Tor Browser are also superior to those of normal Firefox, even when Firefox is used with Torbutton.\n\nTo remove Torbutton, go to Tools->Addons->Extensions and then click the Remove button next to Torbutton. -torbutton.popup.short_torbrowser = Important Torbutton Information!\n\nTorbutton is now always enabled.\n\nClick on the Torbutton for more information. +torbutton.popup.prompt_torbrowser = Torbutton töötab nüüd erinevalt: sa ei saa teda enam välja lülitada.\n\nMe muutsime seda, sest Torbuttoni kasutamine brauseris, mida kasutatakse ka Tor võrgust väljas, ei ole turvaline. Sellega kaasnes liiga palju vigu, mida ei saanud lahendada muud moodi.\n\nKui sa soovid kasutada Firefoxi tavapäraselt, siis eemalda Torbutton ja laadi alla Tor Brauseri pakett. Tor Brauseri privaatsus on samuti parem kui Firefoxil, isegi koos Torbuttoniga.\n\nTorbuttoni eemaldamiseks mine Tööriistad->Lisad->Laiendused ja klõpsa Torbuttoni juures nupul Eemalda +torbutton.popup.short_torbrowser = Tähtis Torbutton'i teave!\n\nTorbutton on nüüd alati lubatud.\n\nLisainfo saamiseks klõpsa Torbuttonil. -torbutton.popup.confirm_plugins = Plugins such as Flash can harm your privacy and anonymity.\n\nThey can also bypass Tor to reveal your current location and IP address.\n\nAre you sure you want to enable plugins?\n\n +torbutton.popup.confirm_plugins = Pistikprogrammid nagu Flash võivad kahjustada su privaatsust ja anonüümsust.\n\nNad saavad Tor'ist mööda minna ja paljastada su praeguse asukoha ja
[tor-commits] [translation/tor-messenger-privdtd_completed] Update translations for tor-messenger-privdtd_completed
commit 126dbff14ac9041c5e6f5ea189a4c3d7d666ce01 Author: Translation commit botDate: Mon May 30 13:46:56 2016 + Update translations for tor-messenger-privdtd_completed --- et/priv.dtd | 2 ++ 1 file changed, 2 insertions(+) diff --git a/et/priv.dtd b/et/priv.dtd new file mode 100644 index 000..c62bb33 --- /dev/null +++ b/et/priv.dtd @@ -0,0 +1,2 @@ + + \ No newline at end of file ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [translation/gettor] Update translations for gettor
commit 1b42a737f2cea0cc4b0d0e2ef27f2a1e9b694181 Author: Translation commit botDate: Mon May 30 13:45:06 2016 + Update translations for gettor --- et/gettor.po | 502 +++ 1 file changed, 502 insertions(+) diff --git a/et/gettor.po b/et/gettor.po new file mode 100644 index 000..f97c3af --- /dev/null +++ b/et/gettor.po @@ -0,0 +1,502 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# +# Translators: +msgid "" +msgstr "" +"Project-Id-Version: The Tor Project\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2013-01-19 13:40+0100\n" +"PO-Revision-Date: 2010-11-30 05:01+\n" +"Last-Translator: FULL NAME \n" +"Language-Team: Estonian (http://www.transifex.com/otf/torproject/language/et/)\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Language: et\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" + +#: lib/gettor/i18n.py:27 +msgid "Hello, This is the \"GetTor\" robot." +msgstr "" + +#: lib/gettor/i18n.py:29 +msgid "Thank you for your request." +msgstr "" + +#: lib/gettor/i18n.py:31 +msgid "" +"Unfortunately, we won't answer you at this address. You should make\n" +"an account with GMAIL.COM, YAHOO.COM or YAHOO.CN and send the mail from\n" +"one of those." +msgstr "" + +#: lib/gettor/i18n.py:35 +msgid "" +"We only process requests from email services that support \"DKIM\",\n" +"which is an email feature that lets us verify that the address in the\n" +"\"From\" line is actually the one who sent the mail." +msgstr "" + +#: lib/gettor/i18n.py:39 +msgid "" +"(We apologize if you didn't ask for this mail. Since your email is from\n" +"a service that doesn't use DKIM, we're sending a short explanation,\n" +"and then we'll ignore this email address for the next day or so.)" +msgstr "" + +#: lib/gettor/i18n.py:43 lib/gettor/i18n.py:135 +msgid "" +"If you have any questions or it doesn't work, you can contact a\n" +"human at this support email address: h...@rt.torproject.org" +msgstr "" + +#: lib/gettor/i18n.py:46 +msgid "" +"I will mail you a Tor package, if you tell me which one you want.\n" +"Please select one of the following package names:\n" +"\n" +"windows\n" +"macos-i386\n" +"macos-ppc\n" +"linux-i386\n" +"linux-x86_64\n" +"obfs-windows\n" +"obfs-macos-i386\n" +"obfs-macos-x86_64\n" +"obfs-linux-i386\n" +"obfs-linux-x86_64\n" +"source" +msgstr "" + +#: lib/gettor/i18n.py:61 +msgid "" +"Please reply to this mail, and tell me a single package name anywhere \n" +"in the body of your email." +msgstr "" + +#: lib/gettor/i18n.py:64 +msgid "" +"OBTAINING LOCALIZED VERSIONS OF TOR\n" +"===" +msgstr "" + +#: lib/gettor/i18n.py:67 +msgid "" +"To get a version of Tor translated into your language, specify the\n" +"language you want in the address you send the mail to:\n" +"\n" +"gettor...@torproject.org" +msgstr "" + +#: lib/gettor/i18n.py:72 +msgid "" +"This example will give you the requested package in a localized\n" +"version for Farsi (Persian). Check below for a list of supported language\n" +"codes. " +msgstr "" + +#: lib/gettor/i18n.py:76 +msgid " List of supported locales:" +msgstr "" + +#: lib/gettor/i18n.py:78 +msgid "Here is a list of all available languages:" +msgstr "" + +#: lib/gettor/i18n.py:80 +msgid "" +"gettor...@torproject.org: Arabic\n" +"gettor...@torproject.org: German\n" +"gettor...@torproject.org: English\n" +"gettor...@torproject.org: Spanish\n" +"gettor...@torproject.org: Farsi (Iran)\n" +"gettor...@torproject.org: French\n" +"gettor...@torproject.org: Italian\n" +"gettor...@torproject.org: Dutch\n" +"gettor...@torproject.org: Polish\n" +"gettor...@torproject.org: Russian\n" +"gettor...@torproject.org: Chinese" +msgstr "" + +#: lib/gettor/i18n.py:92 +msgid "If you select no language, you will receive the English version." +msgstr "" + +#: lib/gettor/i18n.py:94 +msgid "" +"SMALLER SIZED PACKAGES\n" +"==" +msgstr "" + +#: lib/gettor/i18n.py:97 +msgid "" +"If your bandwith is low or your provider doesn't allow you to\n" +"receive large attachments in your email, GetTor can send you several\n" +"small packages instead of one big one." +msgstr "" + +#: lib/gettor/i18n.py:101 +msgid "" +"Simply include the keyword 'split' in a new line on its own (this part\n" +"is important!) like so: \n" +"\n" +"windows\n" +"split" +msgstr "" + +#: lib/gettor/i18n.py:107 +msgid "" +"Sending this text in an email to GetTor will cause it to send you \n" +"the Tor Browser Bundle in a number of 1,4MB attachments." +msgstr "" + +#: lib/gettor/i18n.py:110 +msgid "" +"After having received all parts, you need to re-assemble them to
[tor-commits] [translation/torbutton-brandproperties_completed] Update translations for torbutton-brandproperties_completed
commit cb4b56e13693e9c943e8d309589939ef54fc652e Author: Translation commit botDate: Mon May 30 13:45:55 2016 + Update translations for torbutton-brandproperties_completed --- et/brand.properties | 16 1 file changed, 16 insertions(+) diff --git a/et/brand.properties b/et/brand.properties new file mode 100644 index 000..c40974b --- /dev/null +++ b/et/brand.properties @@ -0,0 +1,16 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +brandShorterName=Tor Brauser +brandShortName=Tor Brauser +brandFullName=Tor Brauser +vendorShortName=Projekt Tor + +homePageSingleStartMain=Firefox Start, kiire koduleht sisseehitatud otsinguga +homePageImport=Impordi oma koduleht %S-st + +homePageMigrationPageTitle=Kodulehe valimine +homePageMigrationDescription=Palun vali koduleht, mida sa soovid kasutada: + +syncBrandShortName=Sünkroniseeri ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [translation/torbutton-branddtd] Update translations for torbutton-branddtd
commit fbb6420217d6ca893597e5f5661cf87ec3660528 Author: Translation commit botDate: Mon May 30 13:45:59 2016 + Update translations for torbutton-branddtd --- et/brand.dtd | 18 +- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/et/brand.dtd b/et/brand.dtd index 3df1a08..a47c121 100644 --- a/et/brand.dtd +++ b/et/brand.dtd @@ -2,14 +2,14 @@ - License, v. 2.0. If a copy of the MPL was not distributed with this - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> - - - - - + + + + + - - - - + + + + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [translation/tails-misc] Update translations for tails-misc
commit b880ec49547b6602a4440cfcb0ff6e45edd1a323 Author: Translation commit botDate: Mon May 30 13:45:45 2016 + Update translations for tails-misc --- et.po | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/et.po b/et.po index 6746ff2..273a531 100644 --- a/et.po +++ b/et.po @@ -8,7 +8,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2016-05-25 20:25+0200\n" -"PO-Revision-Date: 2016-05-27 10:12+\n" +"PO-Revision-Date: 2016-05-30 13:36+\n" "Last-Translator: carolyn \n" "Language-Team: Estonian (http://www.transifex.com/otf/torproject/language/et/)\n" "MIME-Version: 1.0\n" @@ -537,7 +537,7 @@ msgstr "" #: ../config/chroot_local-includes/usr/share/applications/tor-browser.desktop.in.h:1 msgid "Tor Browser" -msgstr "" +msgstr "Tor Brauser" #: ../config/chroot_local-includes/usr/share/applications/tor-browser.desktop.in.h:2 msgid "Anonymous Web Browser" ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [translation/tails-persistence-setup] Update translations for tails-persistence-setup
commit 4e0fdf41d25c607b8c7e5cfd9b150a072ef63e45 Author: Translation commit botDate: Mon May 30 13:45:24 2016 + Update translations for tails-persistence-setup --- et/et.po | 7 --- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/et/et.po b/et/et.po index 40dfda8..d86b84e 100644 --- a/et/et.po +++ b/et/et.po @@ -3,13 +3,14 @@ # This file is distributed under the same license as the PACKAGE package. # # Translators: +# Kristjan Räts , 2016 msgid "" msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: Tails developers \n" "POT-Creation-Date: 2016-04-24 16:40+0200\n" -"PO-Revision-Date: 2016-04-25 08:33+\n" -"Last-Translator: carolyn \n" +"PO-Revision-Date: 2016-05-30 13:45+\n" +"Last-Translator: Kristjan Räts \n" "Language-Team: Estonian (http://www.transifex.com/otf/torproject/language/et/)\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -39,7 +40,7 @@ msgstr "" #: ../lib/Tails/Persistence/Configuration/Presets.pm:70 msgid "SSH keys, configuration and known hosts" -msgstr "" +msgstr "SSH võtmed, häälestus ja tuntud hostid" #: ../lib/Tails/Persistence/Configuration/Presets.pm:78 msgid "Pidgin" ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [translation/torbutton-brandproperties] Update translations for torbutton-brandproperties
commit 5813b520423b0aa11deeb660d82acb4084f04918 Author: Translation commit botDate: Mon May 30 13:45:52 2016 + Update translations for torbutton-brandproperties --- et/brand.properties | 18 +- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/et/brand.properties b/et/brand.properties index 732c157..c40974b 100644 --- a/et/brand.properties +++ b/et/brand.properties @@ -2,15 +2,15 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -brandShorterName=Tor Browser -brandShortName=Tor Browser -brandFullName=Tor Browser -vendorShortName=Tor Project +brandShorterName=Tor Brauser +brandShortName=Tor Brauser +brandFullName=Tor Brauser +vendorShortName=Projekt Tor -homePageSingleStartMain=Firefox Start, a fast home page with built-in search -homePageImport=Import your home page from %S +homePageSingleStartMain=Firefox Start, kiire koduleht sisseehitatud otsinguga +homePageImport=Impordi oma koduleht %S-st -homePageMigrationPageTitle=Home Page Selection -homePageMigrationDescription=Please select the home page you wish to use: +homePageMigrationPageTitle=Kodulehe valimine +homePageMigrationDescription=Palun vali koduleht, mida sa soovid kasutada: -syncBrandShortName=Sync +syncBrandShortName=Sünkroniseeri ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [oonib/master] Document configuration (#78)
commit 0396a55a27e9923537a6cd3d743d37009ef4715b Author: Joe LandersDate: Thu May 26 12:49:23 2016 +0200 Document configuration (#78) * Update README.rst * remove old configuration option --- README.rst | 17 + oonib.conf.example | 1 - 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 63a242c..a73d776 100644 --- a/README.rst +++ b/README.rst @@ -107,6 +107,23 @@ this would simply be ``'report'``) and ``tor_datadir`` (where you would like the spawned Tor process to keep its data). If you compiled Tor yourself, you'll likely want to specify it for the ``tor_binary`` option. +Configure bouncer and collector endpoints +. + +The bouncer and collector are HTTP applications ("protocols" in twisted terminology) that can be configured to run on top of plain TCP, TLS, or onion service endpoints. +Here is an example of the relevant part of the configuration:: +bouncer_endpoints: +- {type: tls, port: 10443, cert: "private/ssl-key-and-cert.pem"} +- {type: tcp, port: 10080} +- {type: onion, hsdir: "/some/private/bouncer"} + +collector_endpoints: +- {type: tls, port: 11443, cert: "private/ssl-key-and-cert.pem"} +- {type: onion, hsdir: "/some/private/collector"} + +`scripts/gen-ssl-key-cert.sh` in this repo contains the openssl command to generate a self-signed certificate which you can use for the tls endpoint. +txtorcon will use the hostname/private_key from the configured hsdir to start an onion service, or generate a new key if hsdir is empty. + Generate self signed certs for OONIB If you want to use the HTTPS test helper, you will need to create a diff --git a/oonib.conf.example b/oonib.conf.example index 95df448..1925d81 100644 --- a/oonib.conf.example +++ b/oonib.conf.example @@ -27,7 +27,6 @@ main: debug: false stale_time: 3600 -tor_hidden_service: false tor_datadir: null bouncer_endpoints: ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [oonib/master] Fix privilege shedding and daemonisation of oonibackend (#80)
commit 5940999a49af50ff49aeabb1103543889c598312 Author: Arturo FilastòDate: Mon May 30 15:24:07 2016 +0200 Fix privilege shedding and daemonisation of oonibackend (#80) * Fix privilege shedding and daemonisation of oonibackend * Rewrite to use twistd * Launch tor only after we have forked * Set the UID of when starting tor This fixes #65 * Add gitignore files and remove empty.txt files * Drop support for super old versions of txtorcon * Remove now unnecessary hacks to maintain backward compatibility * Fix permissions on temporary tor data directory --- bin/oonib | 38 ++- data/archive/.gitignore | 2 + data/archive/empty.txt | 2 - data/decks/.gitignore | 3 + data/inputs/.gitignore | 3 + data/reports/.gitignore | 2 + data/tor/.gitignore | 2 + data/tor/empty.txt | 2 - oonib.conf.example | 4 - oonib/config.py | 2 + oonib/log.py| 61 +-- oonib/onion.py | 63 +-- oonib/oonibackend.py| 280 ++-- requirements.txt| 2 +- setup.py| 6 +- 15 files changed, 213 insertions(+), 259 deletions(-) diff --git a/bin/oonib b/bin/oonib index 109c74e..ccbe246 100755 --- a/bin/oonib +++ b/bin/oonib @@ -2,55 +2,35 @@ import sys import os -from twisted.python import log, usage -from twisted.internet import reactor -from twisted.application import app # Hack to set the proper sys.path. Overcomes the export PYTHONPATH pain. sys.path[:] = map(os.path.abspath, sys.path) sys.path.insert(0, os.path.abspath(os.getcwd())) -from oonib import errors as e +from oonib import errors from oonib.config import config -from oonib.log import LoggerFactory - try: config.load() -except e.ConfigFileNotSpecified: +except errors.ConfigFileNotSpecified: print "Config file not specified!" print "Use -c to specify one!" config.usageOptions() sys.exit(1) -except e.ConfigFileDoesNotExist, path: +except errors.ConfigFileDoesNotExist, path: print "Config file \"%s\" does not exist!" % path sys.exit(2) -except e.InvalidReportDirectory, path: +except errors.InvalidReportDirectory, path: print "Invalid report directory: %s!" % path sys.exit(3) -except e.InvalidArchiveDirectory, path: +except errors.InvalidArchiveDirectory, path: print "Invalid archive directory: %s!" % path sys.exit(4) -except e.InvalidInputDirectory, path: +except errors.InvalidInputDirectory, path: print "Invalid input directory: %s" % path sys.exit(5) -except e.InvalidDeckDirectory, path: +except errors.InvalidDeckDirectory, path: print "Invalid deck directory: %s" % path sys.exit(6) -if config.main.chroot: -sys.argv.append('--chroot') -sys.argv.append(config.chroot) - -if not config.main.nodaemon: -sys.argv.append('-y') - -from oonib.oonibackend import application - -from twisted.scripts._twistd_unix import UnixApplicationRunner -class OBaseRunner(UnixApplicationRunner): -temporary_data_dir = None -def createOrGetApplication(self): -return application - -OBaseRunner.loggerFactory = LoggerFactory -OBaseRunner(config.main).run() +from oonib import oonibackend +oonibackend.start() diff --git a/data/archive/.gitignore b/data/archive/.gitignore new file mode 100644 index 000..d6b7ef3 --- /dev/null +++ b/data/archive/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/data/archive/empty.txt b/data/archive/empty.txt deleted file mode 100644 index 9a166f9..000 --- a/data/archive/empty.txt +++ /dev/null @@ -1,2 +0,0 @@ -In here will end up the archived reports. -This file may be deleted at any time. diff --git a/data/decks/.gitignore b/data/decks/.gitignore new file mode 100644 index 000..514e47a --- /dev/null +++ b/data/decks/.gitignore @@ -0,0 +1,3 @@ +* +!.gitignore +!README diff --git a/data/inputs/.gitignore b/data/inputs/.gitignore new file mode 100644 index 000..a0991ff --- /dev/null +++ b/data/inputs/.gitignore @@ -0,0 +1,3 @@ +* +!.gitignore +!Makefile diff --git a/data/reports/.gitignore b/data/reports/.gitignore new file mode 100644 index 000..d6b7ef3 --- /dev/null +++ b/data/reports/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/data/tor/.gitignore b/data/tor/.gitignore new file mode 100644 index 000..d6b7ef3 --- /dev/null +++ b/data/tor/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/data/tor/empty.txt b/data/tor/empty.txt deleted file mode 100644 index ec3be8f..000 --- a/data/tor/empty.txt +++ /dev/null @@ -1,2 +0,0 @@ -In here will end up all tor related files. -This file may be deleted at any time. diff --git a/oonib.conf.example b/oonib.conf.example index 6f4c7f3..bcf8393 100644 --- a/oonib.conf.example +++ b/oonib.conf.example @@ -7,8 +7,6 @@ main: bouncer_file: data/bouncer.yaml logfile: null -database_uri:
[tor-commits] [oonib/master] Feature/web connectivity (#63)
commit 5178fc299d361c9ab57002e619e40b05061f5a7d Author: Arturo FilastòDate: Sat May 28 19:05:42 2016 +0200 Feature/web connectivity (#63) * Add web connectivity test helper * Forward test helper addresses to the host machine * Align the request headers used by the web_connectivity test helper to those of the probe * Add monkey patch for bug in twisted RedirectAgent: https://twistedmatrix.com/trac/ticket/8265 * Add monkey patch for HTTPClientParser.statusReceived * Use TrueHeaders in control request * Add support for specifying endpoints in web_connectivity test helper * Add endpoint for checking the status of the WebConnectivity test helper * Add support for parsing test-helpers-alternate * Fix key for web_connectivity test in example configuration file * Implement on-disk web_connectivity cache * Add support for Gzip content decoding * Also record CNAME resolution. * Rename ips to addrs * Add support for retries in the http_request * Add support for extracting title * Encode the responses as well when debug mode * Handle partial downloads * Ignore errors when encoding headers * Cast title to unicode and ignore errors * Improvements based on feedback and comments by @bassosimone * Move twisted related patches into txextra module * Add support for returning the responses based on a key sent by the client. * Inherit from OONIBHandler so we can get the error message format * Stylistic improvements * Set defaults in a way that oonib.conf can start from the example * Avoid doing join on nonetype * Address comments by @bassosimone * Properly set the body also when we get a partial body downloaded * Move more code into shared oonib.common module * Fully sync the common module with ooni-probe (pulling in also other shared functionality so the Agents match entirely) * Fix location of comment for the patched HTTPClient * Add unittests and integration tests for web_connectivity --- Vagrantfile | 25 +++ oonib.conf.example | 18 +- oonib/bouncer/handlers.py | 14 +- oonib/common/__init__.py| 10 ++ oonib/common/http_utils.py | 54 ++ oonib/common/tcp_utils.py | 10 ++ oonib/common/txextra.py | 202 + oonib/oonibackend.py| 29 ++- oonib/test/test_web_connectivity.py | 71 oonib/testhelpers/http_helpers.py | 344 +++- 10 files changed, 758 insertions(+), 19 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index b71e4e7..7aa5f71 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -4,6 +4,31 @@ Vagrant.configure("2") do |config| config.vm.box = "precise32" config.vm.box_url = "http://files.vagrantup.com/precise32.box; + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine. In the example below, + # accessing "localhost:8080" will access port 80 on the guest machine. + config.vm.network :forwarded_port, guest: 57001, host: 57001 + config.vm.network :forwarded_port, guest: 57001, host: 57002 + config.vm.network :forwarded_port, guest: 57001, host: 57003 + config.vm.network :forwarded_port, guest: 57004, host: 57004 + config.vm.network :forwarded_port, guest: 57005, host: 57005 + config.vm.network :forwarded_port, guest: 57006, host: 57006 + config.vm.network :forwarded_port, guest: 57007, host: 57007 + + # Create a private network, which allows host-only access to the machine + # using a specific IP. + # config.vm.network :private_network, ip: "192.168.33.10" + + # Create a public network, which generally matched to bridged network. + # Bridged networks make the machine appear as another physical device on + # your network. + # config.vm.network :public_network + + # Share an additional folder to the guest VM. The first argument is + # the path on the host to the actual folder. The second argument is + # the path on the guest to mount the folder. And the optional third + # argument is a set of non-required options. config.vm.synced_folder ".", "/data/oonib" end diff --git a/oonib.conf.example b/oonib.conf.example index 1925d81..6f4c7f3 100644 --- a/oonib.conf.example +++ b/oonib.conf.example @@ -30,12 +30,13 @@ main: tor_datadir: null bouncer_endpoints: -- {type: tls, port: 10443, cert: "private/ssl-key-and-cert.pem"} +#- {type: tls, port: 10443, cert: "private/ssl-key-and-cert.pem"} - {type: tcp, port: 10080} -- {type: onion, hsdir: bouncer} +- {type: onion, hsdir: /tmp/bouncer} collector_endpoints: -- {type: tls, port: 11443, cert:
[tor-commits] [oonib/master] pass tor_binary config to txtorcon (#79)
commit 0b6477c7af96663d2b0d853db66e413cdd599b30 Author: Joe LandersDate: Mon May 23 17:51:52 2016 +0200 pass tor_binary config to txtorcon (#79) --- oonib/onion.py | 8 +--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/oonib/onion.py b/oonib/onion.py index 8abefe5..0a7a155 100644 --- a/oonib/onion.py +++ b/oonib/onion.py @@ -106,13 +106,15 @@ def get_global_tor(reactor): try: if _global_tor_config is None: -_global_tor_config = config = _configTor() +_global_tor_config = _configTor() # start Tor launching def updates(prog, tag, summary): print("%d%%: %s" % (prog, summary)) -yield launch_tor(config, reactor, progress_updates=updates) -yield config.post_bootstrap +yield launch_tor(_global_tor_config, reactor, +progress_updates=updates, +tor_binary=config.main.tor_binary) +yield _global_tor_config.post_bootstrap defer.returnValue(_global_tor_config) finally: ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [oonib/master] only assume HS endpoint when tor_hidden_service: true
commit 35ccea7dbf2b4b3a5d6d72c11f2bde34efd29286 Author: Joe LandersDate: Fri May 13 09:33:24 2016 +0200 only assume HS endpoint when tor_hidden_service: true --- oonib/oonibackend.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index 3802566..eaf235f 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -157,10 +157,10 @@ if config.main.tor_hidden_service: configTor(torconfig) # this is to ensure same behaviour with an old config file -if config.main.bouncer_endpoints is None: +if config.main.bouncer_endpoints is None and config.main.tor_hidden_service: config.main.bouncer_endpoints = [ {'type': 'onion', 'hsdir': 'bouncer'} ] -if config.main.collector_endpoints is None: +if config.main.collector_endpoints is None and config.main.tor_hidden_service: config.main.collector_endpoints = [ {'type': 'onion', 'hsdir': 'collector'} ] for endpoint_config in config.main.bouncer_endpoints: ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [oonib/master] Handle empty bouncer_endpoints and collector_endpoints
commit 9437610ec476c2c20150a905fbeac7e01908c8b7 Author: Arturo FilastòDate: Sat May 21 21:19:52 2016 +0200 Handle empty bouncer_endpoints and collector_endpoints * Set tor_data_dir to null in the default example --- oonib.conf.example | 2 +- oonib/oonibackend.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/oonib.conf.example b/oonib.conf.example index 0a5f4a5..95df448 100644 --- a/oonib.conf.example +++ b/oonib.conf.example @@ -28,7 +28,7 @@ main: stale_time: 3600 tor_hidden_service: false -tor_datadir: test_datadir +tor_datadir: null bouncer_endpoints: - {type: tls, port: 10443, cert: "private/ssl-key-and-cert.pem"} diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index 18bacf0..33ea92d 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -162,12 +162,12 @@ if config.main.tor_hidden_service and \ config.main.bouncer_endpoints = [ {'type': 'onion', 'hsdir': bouncer_hsdir} ] config.main.collector_endpoints = [ {'type': 'onion', 'hsdir': collector_hsdir} ] -for endpoint_config in config.main.get('bouncer_endpoints'): +for endpoint_config in config.main.get('bouncer_endpoints', []): print "Starting bouncer with config %s" % endpoint_config endpoint = getEndpoint(endpoint_config) createService(endpoint, 'bouncer', endpoint_config) -for endpoint_config in config.main.get('collector_endpoints'): +for endpoint_config in config.main.get('collector_endpoints', []): print "Starting collector with config %s" % endpoint_config endpoint = getEndpoint(endpoint_config) createService(endpoint, 'collector', endpoint_config) ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [oonib/master] Ensure bouncer_endpoints and collector_endpoints are by default empty lists
commit 90ca955e970ef30c5963acce77c5f736372fc440 Author: Arturo FilastòDate: Fri May 13 13:20:28 2016 +0200 Ensure bouncer_endpoints and collector_endpoints are by default empty lists --- oonib/oonibackend.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index 5a994e8..337b797 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -163,6 +163,9 @@ if config.main.bouncer_endpoints is None and config.main.tor_hidden_service: if config.main.collector_endpoints is None and config.main.tor_hidden_service: config.main.collector_endpoints = [ {'type': 'onion', 'hsdir': 'collector'} ] +config.main.bouncer_endpoints = config.main.get('bouncer_endpoints', []) +config.main.collector_endpoints = config.main.get('collector_endpoints', []) + for endpoint_config in config.main.bouncer_endpoints: print "Starting bouncer with config %s" % endpoint_config endpoint = getEndpoint(endpoint_config) @@ -172,4 +175,3 @@ for endpoint_config in config.main.collector_endpoints: print "Starting collector with config %s" % endpoint_config endpoint = getEndpoint(endpoint_config) createService(endpoint, 'collector', endpoint_config) - ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [oonib/master] [hotfix] Only create bouncer endpoints when it's configured
commit f03f8055d44d3bbb7cbcfb789946ba67a94e1e52 Author: Arturo FilastòDate: Thu May 26 13:49:28 2016 +0200 [hotfix] Only create bouncer endpoints when it's configured --- oonib/oonibackend.py | 9 ++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index 33ea92d..601a972 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -163,9 +163,12 @@ if config.main.tor_hidden_service and \ config.main.collector_endpoints = [ {'type': 'onion', 'hsdir': collector_hsdir} ] for endpoint_config in config.main.get('bouncer_endpoints', []): -print "Starting bouncer with config %s" % endpoint_config -endpoint = getEndpoint(endpoint_config) -createService(endpoint, 'bouncer', endpoint_config) +if config.main.bouncer_file: +print "Starting bouncer with config %s" % endpoint_config +endpoint = getEndpoint(endpoint_config) +createService(endpoint, 'bouncer', endpoint_config) +else: +print "No bouncer configured" for endpoint_config in config.main.get('collector_endpoints', []): print "Starting collector with config %s" % endpoint_config ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [oonib/master] Map also the connect_error (#85)
commit 2b92f6327fc2bcb4da06e686fbc3b5b4fc4bbbe6 Author: Arturo FilastòDate: Mon May 30 15:39:05 2016 +0200 Map also the connect_error (#85) --- oonib/testhelpers/http_helpers.py | 9 - 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/oonib/testhelpers/http_helpers.py b/oonib/testhelpers/http_helpers.py index 1cddf24..a771a19 100644 --- a/oonib/testhelpers/http_helpers.py +++ b/oonib/testhelpers/http_helpers.py @@ -11,8 +11,10 @@ from cyclone.web import RequestHandler, Application, HTTPError from cyclone.web import asynchronous from twisted.internet import protocol, defer, reactor from twisted.internet.endpoints import TCP4ClientEndpoint -from twisted.internet.error import ConnectionRefusedError + +from twisted.internet.error import ConnectionRefusedError, ConnectError from twisted.internet.error import DNSLookupError, TimeoutError + from twisted.names import client as dns_client from twisted.names import dns from twisted.names.error import DNSNameError, DNSServerError @@ -359,6 +361,8 @@ class WebConnectivityCache(object): page_info['failure'] = 'generic_timeout_error' except ConnectionRefusedError: page_info['failure'] = 'connection_refused_error' +except ConnectError: +page_info['failure'] = 'connect_error' except: # XXX map more failures page_info['failure'] = 'unknown_error' @@ -390,6 +394,9 @@ class WebConnectivityCache(object): except ConnectionRefusedError: socket_info['status'] = False socket_info['failure'] = 'connection_refused_error' +except ConnectError: +socket_info['status'] = False +socket_info['failure'] = 'connect_error' except: socket_info['status'] = False socket_info['failure'] = 'unknown_error' ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [oonib/master] Vagrant: use box canonical name rather than URL (#84)
commit 973c9b0c2ae089e3043253890148d25dd550f756 Author: Simone BassoDate: Mon May 30 10:50:21 2016 +0200 Vagrant: use box canonical name rather than URL (#84) This allows us to get updates running `vagrant box update`. While at it, add `/.vagrant/` to `.gitignore`. --- .gitignore | 3 +++ Vagrantfile | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 7fc5cbd..3f4029e 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,6 @@ installed-files.txt oonib.conf oonib.pid + +# Local vagrant files +/.vagrant/ diff --git a/Vagrantfile b/Vagrantfile index 7aa5f71..a2abe0d 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -2,8 +2,7 @@ # vi: set ft=ruby : Vagrant.configure("2") do |config| - config.vm.box = "precise32" - config.vm.box_url = "http://files.vagrantup.com/precise32.box; + config.vm.box = "ubuntu/precise32" # Create a forwarded port mapping which allows access to a specific port # within the machine from a port on the host machine. In the example below, ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [oonib/master] keep old config behavior with new endpoint code
commit 575f0aa5a44d1f851fb3d4fbb1bcf1fbf36a4f2d Author: Joe LandersDate: Sun May 15 18:25:41 2016 +0200 keep old config behavior with new endpoint code --- oonib/onion.py | 36 oonib/oonibackend.py | 42 +++--- 2 files changed, 51 insertions(+), 27 deletions(-) diff --git a/oonib/onion.py b/oonib/onion.py index 262e672..8abefe5 100644 --- a/oonib/onion.py +++ b/oonib/onion.py @@ -1,7 +1,7 @@ import tempfile from oonib import log from oonib.config import config -from twisted.internet import reactor, endpoints +from twisted.internet import reactor, endpoints, defer import os from random import randint @@ -53,9 +53,8 @@ def txSetupFailed(failure): log.err("Setup failed") log.exception(failure) -def configTor(torconfig): -def updates(prog, tag, summary): -print("%d%%: %s" % (prog, summary)) +def _configTor(): +torconfig = TorConfig() if config.main.socks_port: torconfig.SocksPort = config.main.socks_port @@ -89,3 +88,32 @@ def configTor(torconfig): config.main.socks_port = socks_port torconfig.save() +return torconfig + +# get_global_tor is a near-rip of that from txtorcon (so you can have some +# confidence in the logic of it), but we use our own _configTor() while +# the txtorcon function hardcodes some default values we don't want. +_global_tor_config = None +_global_tor_lock = defer.DeferredLock() +# we need the lock because we (potentially) yield several times while +# "creating" the TorConfig instance + +@defer.inlineCallbacks +def get_global_tor(reactor): +global _global_tor_config +global _global_tor_lock +yield _global_tor_lock.acquire() + +try: +if _global_tor_config is None: +_global_tor_config = config = _configTor() + +# start Tor launching +def updates(prog, tag, summary): +print("%d%%: %s" % (prog, summary)) +yield launch_tor(config, reactor, progress_updates=updates) +yield config.post_bootstrap + +defer.returnValue(_global_tor_config) +finally: +_global_tor_lock.release() diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index 337b797..18bacf0 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -11,14 +11,14 @@ from distutils.version import LooseVersion from oonib.api import ooniBackend, ooniBouncer from oonib.config import config -from oonib.onion import configTor +from oonib.onion import get_global_tor from oonib.testhelpers import dns_helpers, ssl_helpers from oonib.testhelpers import http_helpers, tcp_helpers import os from twisted.application import internet, service -from twisted.internet import reactor, endpoints, ssl +from twisted.internet import reactor, endpoints, ssl, defer from twisted.names import dns from txtorcon import TCPHiddenServiceEndpoint, TorConfig @@ -103,15 +103,17 @@ if config.helpers['http-return-json-headers'].port: http_return_request_helper.startService() def getHSEndpoint(endpoint_config): -if torconfig is None: -raise Exception("you probably need to set tor_hidden_service: true") -hsdir = os.path.join(torconfig.DataDirectory, endpoint_config['hsdir']) +hsdir = endpoint_config['hsdir'] +hsdir = os.path.expanduser(hsdir) +hsdir = os.path.realpath(hsdir) if LooseVersion(txtorcon_version) >= LooseVersion('0.10.0'): -return TCPHiddenServiceEndpoint.global_tor(reactor, +return TCPHiddenServiceEndpoint(reactor, +get_global_tor(reactor), 80, hidden_service_dir=hsdir) else: -return TCPHiddenServiceEndpoint.global_tor(reactor, +return TCPHiddenServiceEndpoint(reactor, +get_global_tor(reactor), 80, data_dir=hsdir) @@ -151,27 +153,21 @@ def createService(endpoint, role, endpoint_config): multiService.addService(service) service.startService() -torconfig = None -if config.main.tor_hidden_service: -torconfig = TorConfig() -configTor(torconfig) - # this is to ensure same behaviour with an old config file -if config.main.bouncer_endpoints is None and config.main.tor_hidden_service: -config.main.bouncer_endpoints = [ {'type': 'onion', 'hsdir': 'bouncer'} ] - -if config.main.collector_endpoints is None and config.main.tor_hidden_service: -config.main.collector_endpoints = [ {'type': 'onion', 'hsdir': 'collector'} ] - -config.main.bouncer_endpoints = config.main.get('bouncer_endpoints', []) -config.main.collector_endpoints = config.main.get('collector_endpoints', []) - -for endpoint_config in config.main.bouncer_endpoints: +if config.main.tor_hidden_service and \ +config.main.bouncer_endpoints
[tor-commits] [oonib/master] Fix typo in gen-ssl-key-cert script
commit d7f9a4c731acd9979376fa73438d4e8afae75446 Author: Arturo FilastòDate: Fri May 13 12:57:38 2016 +0200 Fix typo in gen-ssl-key-cert script --- scripts/gen-ssl-key-cert.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gen-ssl-key-cert.sh b/scripts/gen-ssl-key-cert.sh index 2a50766..da85b58 100755 --- a/scripts/gen-ssl-key-cert.sh +++ b/scripts/gen-ssl-key-cert.sh @@ -10,4 +10,4 @@ openssl req -x509 -newkey rsa:2048 \ -keyout private/ssl-key.pem -out private/ssl-cert.pem \ -days 400 -nodes -subj '/CN=selfie' -cat private/ssl-key.pem private/ssh-cert.pem > private/ssl-key-and-cert.pem +cat private/ssl-key.pem private/ssl-cert.pem > private/ssl-key-and-cert.pem ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [oonib/master] Remove unused imports
commit 0bd12a10549d214fff93c4b11e14b5d62bafc0ea Author: Arturo FilastòDate: Fri May 13 13:18:05 2016 +0200 Remove unused imports --- oonib/oonibackend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index eaf235f..5a994e8 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -18,7 +18,7 @@ from oonib.testhelpers import http_helpers, tcp_helpers import os from twisted.application import internet, service -from twisted.internet import reactor, endpoints, defer, ssl, protocol +from twisted.internet import reactor, endpoints, ssl from twisted.names import dns from txtorcon import TCPHiddenServiceEndpoint, TorConfig ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [oonib/master] Merge branch 'feature/https_endpoint'
commit 612eb5a1c7a6ea1f2f83d3eb7a9f5ffb1d68c64f Merge: 94d3452 90ca955 Author: Arturo FilastòDate: Fri May 13 13:21:28 2016 +0200 Merge branch 'feature/https_endpoint' * feature/https_endpoint: Ensure bouncer_endpoints and collector_endpoints are by default empty lists Remove unused imports Fix typo in gen-ssl-key-cert script only assume HS endpoint when tor_hidden_service: true explain + bail out when inconsistent config Move gen-ssl-key-cert to scripts directory remove another print from debugging maintain old behaviour remove echo server i used for debugging easily swappable {TCP, TLS, onion} endpoints oonib.conf.example | 13 - oonib/onion.py | 10 +--- oonib/oonibackend.py| 130 +--- scripts/gen-ssl-key-cert.sh | 13 + 4 files changed, 98 insertions(+), 68 deletions(-) ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [oonib/master] Download openssl with wget as well to fix SSL errors on MLAB
commit 94d34521e9ec7f6a61b1d486dbc53506cb9034cb Author: Arturo FilastòDate: Tue May 10 19:06:44 2016 +0200 Download openssl with wget as well to fix SSL errors on MLAB --- scripts/build_tor2web_tor.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_tor2web_tor.sh b/scripts/build_tor2web_tor.sh index 080a878..c9d7d1b 100755 --- a/scripts/build_tor2web_tor.sh +++ b/scripts/build_tor2web_tor.sh @@ -12,12 +12,12 @@ SCRIPT_ROOT=`pwd` # Package URLS CURL_URLS="\ http://zlib.net/zlib-$ZLIB_VERSION.tar.gz -https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz.asc -https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz https://github.com/downloads/libevent/libevent/libevent-$LIBEVENT_VERSION.tar.gz.asc https://github.com/downloads/libevent/libevent/libevent-$LIBEVENT_VERSION.tar.gz; WGET_URLS="\ +https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz.asc +https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz https://dist.torproject.org/tor-$TOR_VERSION.tar.gz https://dist.torproject.org/tor-$TOR_VERSION.tar.gz.asc; ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [oonib/master] maintain old behaviour
commit 5fa24c3364c4349995d3bb1ca900dc54e546f5c4 Author: Joe LandersDate: Fri May 6 13:11:33 2016 +0200 maintain old behaviour --- oonib/oonibackend.py | 27 --- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index 0b8c4ec..1e77d6b 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -154,15 +154,20 @@ if config.main.tor_hidden_service: torconfig = TorConfig() configTor(torconfig) -if config.main.bouncer_endpoints: -for endpoint_config in config.main.bouncer_endpoints: -print "Starting bouncer with config %s" % endpoint_config -endpoint = getEndpoint(endpoint_config) -createService(endpoint, 'bouncer', endpoint_config) - -if config.main.bouncer_endpoints: -for endpoint_config in config.main.collector_endpoints: -print "Starting collector with config %s" % endpoint_config -endpoint = getEndpoint(endpoint_config) -createService(endpoint, 'collector', endpoint_config) +# this is to ensure same behaviour with an old config file +if config.main.bouncer_endpoints is None: +config.main.bouncer_endpoints = [ {'type': 'onion', 'hsdir': 'bouncer'} ] + +if config.main.collector_endpoints is None: +config.main.collector_endpoints = [ {'type': 'onion', 'hsdir': 'collector'} ] + +for endpoint_config in config.main.bouncer_endpoints: +print "Starting bouncer with config %s" % endpoint_config +endpoint = getEndpoint(endpoint_config) +createService(endpoint, 'bouncer', endpoint_config) + +for endpoint_config in config.main.collector_endpoints: +print "Starting collector with config %s" % endpoint_config +endpoint = getEndpoint(endpoint_config) +createService(endpoint, 'collector', endpoint_config) ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits