[tor-commits] [metrics-lib/master] Parse new NAT-based Snowflake lines.
commit 8976cdd9be1bb70d3457bf2551971e12ee253ce1
Author: Karsten Loesing
Date: Fri Dec 18 12:02:55 2020 +0100
Parse new NAT-based Snowflake lines.
Implements #40002.
---
CHANGELOG.md | 5 +-
.../org/torproject/descriptor/SnowflakeStats.java | 54
.../java/org/torproject/descriptor/impl/Key.java | 5 ++
.../descriptor/impl/SnowflakeStatsImpl.java| 75 ++
.../descriptor/impl/SnowflakeStatsImplTest.java| 50 +++
5 files changed, 188 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ac865a9..68ad2a2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,7 @@
-# Changes in version 2.??.? - 2020-??-??
+# Changes in version 2.16.0 - 2020-??-??
+
+ * Medium changes
+ - Parse new NAT-based Snowflake lines.
# Changes in version 2.15.0 - 2020-12-11
diff --git a/src/main/java/org/torproject/descriptor/SnowflakeStats.java
b/src/main/java/org/torproject/descriptor/SnowflakeStats.java
index 2fe78df..967d061 100644
--- a/src/main/java/org/torproject/descriptor/SnowflakeStats.java
+++ b/src/main/java/org/torproject/descriptor/SnowflakeStats.java
@@ -103,6 +103,30 @@ public interface SnowflakeStats extends Descriptor {
*/
Optional clientDeniedCount();
+ /**
+ * Return a count of the number of times a client with a restricted or
unknown
+ * NAT type has requested a proxy from the broker but no proxies were
+ * available, rounded up to the nearest multiple of 8.
+ *
+ * @return Count of the number of times a client with a restricted or unknown
+ * NAT type has requested a proxy from the broker but no proxies were
+ * available, rounded up to the nearest multiple of 8.
+ * @since 2.16.0
+ */
+ Optional clientRestrictedDeniedCount();
+
+ /**
+ * Return a count of the number of times a client with an unrestricted NAT
+ * type has requested a proxy from the broker but no proxies were available,
+ * rounded up to the nearest multiple of 8.
+ *
+ * @return Count of the number of times a client with an unrestricted NAT
type
+ * has requested a proxy from the broker but no proxies were available,
+ * rounded up to the nearest multiple of 8.
+ * @since 2.16.0
+ */
+ Optional clientUnrestrictedDeniedCount();
+
/**
* Return a count of the number of times a client successfully received a
* proxy from the broker, rounded up to the nearest multiple of 8.
@@ -112,5 +136,35 @@ public interface SnowflakeStats extends Descriptor {
* @since 2.7.0
*/
Optional clientSnowflakeMatchCount();
+
+ /**
+ * Return a count of the total number of unique IP addresses of snowflake
+ * proxies that have a restricted NAT type.
+ *
+ * @return Count of the total number of unique IP addresses of snowflake
+ * proxies that have a restricted NAT type.
+ * @since 2.16.0
+ */
+ Optional snowflakeIpsNatRestricted();
+
+ /**
+ * Return a count of the total number of unique IP addresses of snowflake
+ * proxies that have an unrestricted NAT type.
+ *
+ * @return Count of the total number of unique IP addresses of snowflake
+ * proxies that have an unrestricted NAT type.
+ * @since 2.16.0
+ */
+ Optional snowflakeIpsNatUnrestricted();
+
+ /**
+ * Return a count of the total number of unique IP addresses of snowflake
+ * proxies that have an unknown NAT type.
+ *
+ * @return Count of the total number of unique IP addresses of snowflake
+ * proxies that have an unknown NAT type.
+ * @since 2.16.0
+ */
+ Optional snowflakeIpsNatUnknown();
}
diff --git a/src/main/java/org/torproject/descriptor/impl/Key.java
b/src/main/java/org/torproject/descriptor/impl/Key.java
index b02d96e..410cef6 100644
--- a/src/main/java/org/torproject/descriptor/impl/Key.java
+++ b/src/main/java/org/torproject/descriptor/impl/Key.java
@@ -36,7 +36,9 @@ public enum Key {
CELL_STATS_END("cell-stats-end"),
CELL_TIME_IN_QUEUE("cell-time-in-queue"),
CLIENT_DENIED_COUNT("client-denied-count"),
+ CLIENT_RESTRICTED_DENIED_COUNT("client-restricted-denied-count"),
CLIENT_SNOWFLAKE_MATCH_COUNT("client-snowflake-match-count"),
+ CLIENT_UNRESTRICTED_DENIED_COUNT("client-unrestricted-denied-count"),
CLIENT_VERSIONS("client-versions"),
CONN_BI_DIRECT("conn-bi-direct"),
CONSENSUS_METHOD("consensus-method"),
@@ -149,6 +151,9 @@ public enum Key {
SNOWFLAKE_IDLE_COUNT("snowflake-idle-count"),
SNOWFLAKE_IPS("snowflake-ips"),
SNOWFLAKE_IPS_BADGE("snowflake-ips-badge"),
+ SNOWFLAKE_IPS_NAT_RESTRICTED("snowflake-ips-nat-restricted"),
+ SNOWFLAKE_IPS_NAT_UNKNOWN("snowflake-ips-nat-unknown"),
+ SNOWFLAKE_IPS_NAT_UNRESTRICTED("snowflake-ips-nat-unrestricted"),
SNOWFLAKE_IPS_STANDALONE("snowflake-ips-standalone&qu
[tor-commits] [metrics-web/master] Update to metrics-lib 2.15.0.
commit a07829a30bb85864f46f8668882689d16382ad6a Author: Karsten Loesing Date: Thu Dec 17 15:27:02 2020 +0100 Update to metrics-lib 2.15.0. --- build.xml | 2 +- src/submods/metrics-lib | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.xml b/build.xml index 0d82a81..8cfc81a 100644 --- a/build.xml +++ b/build.xml @@ -10,7 +10,7 @@ - + https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/master] Update instructions when javascript is disabled.
commit 07f70b1ce967585c2307ea1170f85a9274b04569 Author: Karsten Loesing Date: Thu Dec 17 15:29:11 2020 +0100 Update instructions when javascript is disabled. Fixes #31714. --- src/main/resources/web/jsps/rs.jsp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/web/jsps/rs.jsp b/src/main/resources/web/jsps/rs.jsp index ec635f7..281f932 100644 --- a/src/main/resources/web/jsps/rs.jsp +++ b/src/main/resources/web/jsps/rs.jsp @@ -29,7 +29,7 @@ JavaScript required - Please enable JavaScript to use this service. If you are using Tor Browser on High Security mode, it is possible to enable JavaScript to run only on this page. Click the NoScript icon on your address bar and select "Temporarily allow all on this page". Relay Search only uses JavaScript resources that are hosted by the Tor Metrics team. + Please enable JavaScript to use this service. If you are using Tor Browser on Safest mode, you'll have to switch to Safer or Standard mode. Relay Search only uses JavaScript resources that are hosted by the Tor Metrics team. ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/master] Update news.json.
commit 5df9d4c9115b4d289ee1cb428f0e433549b88718
Author: Karsten Loesing
Date: Thu Dec 17 15:31:35 2020 +0100
Update news.json.
---
src/main/resources/web/json/news.json | 119 +-
1 file changed, 118 insertions(+), 1 deletion(-)
diff --git a/src/main/resources/web/json/news.json
b/src/main/resources/web/json/news.json
index 901b00c..b788fe5 100644
--- a/src/main/resources/web/json/news.json
+++ b/src/main/resources/web/json/news.json
@@ -1,4 +1,56 @@
[ {
+ "start" : "2020-12-05",
+ "end" : "2020-12-09",
+ "protocols" : [ "meek", "moat" ],
+ "short_description" : "An update to an Azure CDN edge server TLS certificate
causes an outage of meek and Moat",
+ "description" : "An update to an Azure CDN edge server TLS certificate
causes an outage of meek and Moat. It is fixed by the release of Tor Browser
10.0.6, which updates the built-in public key pinning of obfs4proxy.",
+ "links" : [ {
+"label" : "ticket",
+"target" :
"https://bugs.torproject.org/tpo/anti-censorship/pluggable-transports/meek/40001;
+ }, {
+"label" : "blog post",
+"target" :
"https://blog.torproject.org/new-release-tor-browser-1006#comments;
+ }, {
+"label" : "meek graph",
+"target" :
"https://metrics.torproject.org/userstats-bridge-transport.html?start=2020-11-01=2020-12-31=meek;
+ }, {
+"label" : "BridgeDB graph",
+"target" :
"https://metrics.torproject.org/bridgedb-distributor.html?start=2020-11-01=2020-12-31;
+ }, {
+"label" : "certificate transparency",
+"target" :
"https://crt.sh/?q=cd29bc427d93bc4453d129a294cfd5e082eacbf3fe9a19b7718a50422b6e6cc5;
+ } ]
+}, {
+ "start" : "2020-11-08",
+ "end" : "2020-11-09",
+ "protocols" : [ "snowflake" ],
+ "short_description" : "Outage of the snowflake bridge",
+ "description" : "Outage of the snowflake bridge",
+ "links" : [ {
+"label" : "ticket",
+"target" :
"https://bugs.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/40020;
+ } ]
+}, {
+ "start" : "2020-10-27",
+ "ongoing" : true,
+ "places" : [ "tz" ],
+ "protocols" : [ "", "obfs4" ],
+ "short_description" : "Blocking of Tor and default obfs4 bridges during
elections in Tanzania.",
+ "description" : "Blocking of Tor and default obfs4 bridges during elections
in Tanzania.",
+ "links" : [ {
+"label" : "OONI report",
+"target" :
"https://ooni.org/post/2020-tanzania-blocks-social-media-tor-election-day/#blocking-of-tor;
+ } ]
+}, {
+ "start" : "2020-10-05",
+ "protocols" : [ "snowflake" ],
+ "short_description" : "Deployed version 0.4.2 of the Snowflake WebExtension,
with better counting of clients.",
+ "description" : "Deployed version 0.4.2 of the Snowflake WebExtension, with
better counting of clients.",
+ "links" : [ {
+"label" : "comment",
+"target" :
"https://bugs.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/33157#note_2710701;
+ } ]
+}, {
"start" : "2020-08-31",
"protocols" : [ "obfs4" ],
"short_description" : "Shutdown of default obfs4 bridge frosty.",
@@ -16,6 +68,19 @@
"label" : "comment",
"target" :
"https://bugs.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake-webext/13#note_2705805;
} ]
+}, {
+ "start" : "2020-08-09",
+ "end" : "2020-08-12",
+ "places" : [ "by" ],
+ "short_description" : "Internet shutdowns in Belarus, during protests
following a presidential election.",
+ "description" : "Internet shutdowns in Belarus, during protests following a
presidential election.",
+ "links" : [ {
+"label" : "report",
+"target" :
"https://ooni.org/post/2020-belarus-internet-outages-website-censorship/#internet-outages-amid-2020-belarusian-presidential-election;
+ }, {
+"label" : "thread",
+"target" : "https://github.com/net4people/bbs/issues/46;
+ } ]
}, {
"start" : "2020-07-06",
"protocols" : [ "snowflake" ],
@@ -347,6 +412,9 @@
"links" : [ {
[tor-commits] [collector/master] Include OnionPerf analysis files when syncing.
commit 06c0d78e4a73042c0e7fb6052f079237b8d7ff01
Author: Karsten Loesing
Date: Mon Dec 14 23:23:54 2020 +0100
Include OnionPerf analysis files when syncing.
---
CHANGELOG.md | 4 +-
.../collector/onionperf/OnionPerfDownloader.java | 1 +
.../collector/persist/OnionPerfPersistence.java| 60 +++---
src/main/resources/collector.properties| 2 +-
4 files changed, 57 insertions(+), 10 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f56f74d..7956e55 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,8 +4,8 @@
- Clean up descriptors written to the `out/` directory by deleting
files that are older than seven weeks.
- Correctly index files that are moved away and back.
- - Include microdescriptors and certs when syncing from another
-CollecTor instance.
+ - Include microdescriptors, certs, and OnionPerf analysis files when
+syncing from another CollecTor instance.
- Update to metrics-lib 2.15.0.
diff --git
a/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java
b/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java
index f90bdfe..352d24a 100644
---
a/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java
+++
b/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java
@@ -57,6 +57,7 @@ public class OnionPerfDownloader extends CollecTorMain {
public OnionPerfDownloader(Configuration config) {
super(config);
this.mapPathDescriptors.put("recent/torperf", TorperfResult.class);
+this.mapPathDescriptors.put("recent/onionperf", TorperfResult.class);
}
/** File containing the download history, which is necessary, because
diff --git
a/src/main/java/org/torproject/metrics/collector/persist/OnionPerfPersistence.java
b/src/main/java/org/torproject/metrics/collector/persist/OnionPerfPersistence.java
index 7ed16a2..8975d80 100644
---
a/src/main/java/org/torproject/metrics/collector/persist/OnionPerfPersistence.java
+++
b/src/main/java/org/torproject/metrics/collector/persist/OnionPerfPersistence.java
@@ -6,12 +6,21 @@ package org.torproject.metrics.collector.persist;
import org.torproject.descriptor.TorperfResult;
import org.torproject.metrics.collector.conf.Annotation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
public class OnionPerfPersistence
extends DescriptorPersistence {
+ private static final Logger logger
+ = LoggerFactory.getLogger(OnionPerfPersistence.class);
+
private static final String ONIONPERF = "torperf";
public OnionPerfPersistence(TorperfResult desc) {
@@ -32,18 +41,55 @@ public class OnionPerfPersistence
name).toString();
}
- /** OnionPerf default storage appends. */
+ /** If the original descriptor file was a .tpf file, append the parsed
Torperf
+ * result to the destination .tpf file, but if it was a .json.xz file, just
+ * copy over the entire file, unless it already exists. */
@Override
- public boolean storeOut(String outRoot) {
-return super.storeOut(outRoot, StandardOpenOption.APPEND);
+ public boolean storeOut(String outRoot, StandardOpenOption option) {
+if (desc.getDescriptorFile().getName().endsWith(".tpf")) {
+ return super.storeOut(outRoot, StandardOpenOption.APPEND);
+} else {
+ String fileName = desc.getDescriptorFile().getName();
+ String[] dateParts = fileName.split("\\.")[0].split("-");
+ return this.copyIfNotExists(
+ Paths.get(outRoot,
+ "onionperf",
+ dateParts[0], // year
+ dateParts[1], // month
+ dateParts[2], // day
+ fileName));
+}
}
- /** OnionPerf default storage appends. */
+ /** If the original descriptor file was a .tpf file, append the parsed
Torperf
+ * result to the destination .tpf file, but if it was a .json.xz file, just
+ * copy over the entire file, unless it already exists. */
@Override
- public boolean storeAll(String recentRoot, String outRoot) {
-return super.storeAll(recentRoot, outRoot, StandardOpenOption.APPEND,
-StandardOpenOption.APPEND);
+ public boolean storeRecent(String recentRoot, StandardOpenOption option) {
+if (desc.getDescriptorFile().getName().endsWith(".tpf")) {
+ return super.storeRecent(recentRoot, StandardOpenOption.APPEND);
+} else {
+ String fileName = desc.getDescriptorFile().getName();
+ return this.copyIfNotExists(
+ Paths.get(recentRoot,
+ "onionperf",
+ fileName));
+}
}
+ private boolean copyIfNotExists(Path destinationFile) {
+if (Files.exists(destinationFile)) {
[tor-commits] [collector/master] Include certs when syncing from another instance.
commit 9834cec8c0e88c1e810434f543f5d8c4d5cb2de8
Author: Karsten Loesing
Date: Mon Dec 14 09:35:35 2020 +0100
Include certs when syncing from another instance.
---
CHANGELOG.md | 4 ++--
.../DirectoryKeyCertificatePersistence.java| 27 ++
.../collector/relaydescs/ArchiveWriter.java| 7 +-
.../metrics/collector/sync/SyncPersistence.java| 6 +
4 files changed, 41 insertions(+), 3 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 22d3517..f56f74d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,8 +4,8 @@
- Clean up descriptors written to the `out/` directory by deleting
files that are older than seven weeks.
- Correctly index files that are moved away and back.
- - Include microdescriptors when syncing from another CollecTor
-instance.
+ - Include microdescriptors and certs when syncing from another
+CollecTor instance.
- Update to metrics-lib 2.15.0.
diff --git
a/src/main/java/org/torproject/metrics/collector/persist/DirectoryKeyCertificatePersistence.java
b/src/main/java/org/torproject/metrics/collector/persist/DirectoryKeyCertificatePersistence.java
new file mode 100644
index 000..39f88f3
--- /dev/null
+++
b/src/main/java/org/torproject/metrics/collector/persist/DirectoryKeyCertificatePersistence.java
@@ -0,0 +1,27 @@
+/* Copyright 2020 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.metrics.collector.persist;
+
+import org.torproject.descriptor.DirectoryKeyCertificate;
+import org.torproject.metrics.collector.conf.Annotation;
+
+import java.nio.file.Paths;
+
+public class DirectoryKeyCertificatePersistence
+extends DescriptorPersistence {
+
+ public DirectoryKeyCertificatePersistence(
+ DirectoryKeyCertificate descriptor) {
+super(descriptor, Annotation.Cert.bytes());
+this.calculatePaths();
+ }
+
+ private void calculatePaths() {
+String fileName = this.desc.getFingerprint().toUpperCase() + "-"
++ PersistenceUtils.dateTime(this.desc.getDirKeyPublishedMillis());
+this.recentPath = Paths.get(RELAYDESCS, "certs", fileName).toString();
+this.storagePath = this.recentPath;
+ }
+}
+
diff --git
a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
index 5c58f23..616d7dd 100644
---
a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
+++
b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
@@ -7,6 +7,7 @@ import org.torproject.descriptor.BandwidthFile;
import org.torproject.descriptor.Descriptor;
import org.torproject.descriptor.DescriptorParser;
import org.torproject.descriptor.DescriptorSourceFactory;
+import org.torproject.descriptor.DirectoryKeyCertificate;
import org.torproject.descriptor.Microdescriptor;
import org.torproject.descriptor.RelayExtraInfoDescriptor;
import org.torproject.descriptor.RelayNetworkStatusConsensus;
@@ -105,6 +106,8 @@ public class ArchiveWriter extends CollecTorMain {
super(config);
this.mapPathDescriptors.put("recent/relay-descriptors/votes",
RelayNetworkStatusVote.class);
+this.mapPathDescriptors.put("recent/relay-descriptors/certs",
+DirectoryKeyCertificate.class);
this.mapPathDescriptors.put("recent/relay-descriptors/consensuses",
RelayNetworkStatusConsensus.class);
this.mapPathDescriptors.put(
@@ -738,7 +741,9 @@ public class ArchiveWriter extends CollecTorMain {
"-MM-dd-HH-mm-ss");
File tarballFile = Paths.get(this.outputDirectory, "certs",
fingerprint + "-" + printFormat.format(new Date(published))).toFile();
-File[] outputFiles = new File[] { tarballFile };
+File rsyncFile = Paths.get(recentPathName, RELAY_DESCRIPTORS, "certs",
+tarballFile.getName()).toFile();
+File[] outputFiles = new File[] { tarballFile, rsyncFile };
if (this.store(Annotation.Cert.bytes(), data, outputFiles, null)) {
this.storedCertsCounter++;
}
diff --git
a/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java
b/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java
index e8f780e..ba06bd1 100644
--- a/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java
+++ b/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java
@@ -10,6 +10,7 @@ import org.torproject.descriptor.BridgePoolAssignment;
import org.torproject.descriptor.BridgeServerDescriptor;
import org.torproject.descriptor.BridgedbMetrics;
import org.torproject.descriptor.Descriptor;
+import org.torproject.descriptor.DirectoryKeyCertificate;
import org.torproject.descriptor.ExitList;
import org.torproject.descriptor.Microdescri
[tor-commits] [collector/master] Make sure that the DirectoryStream gets closed.
commit 83850daa9de893cf2d7f846e67191a469cc64900
Author: Karsten Loesing
Date: Sat Dec 5 22:22:22 2020 +0100
Make sure that the DirectoryStream gets closed.
As the docs say, "If timely disposal of file system resources is
required, the try-with-resources construct should be used to ensure
that the stream's close method is invoked after the stream operations
are completed."
Turns out that without closing the stream, the JVM runs out of memory
pretty quickly. Doing this is not optional but mandatory.
---
.../torproject/metrics/collector/persist/PersistenceUtils.java | 10 +++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git
a/src/main/java/org/torproject/metrics/collector/persist/PersistenceUtils.java
b/src/main/java/org/torproject/metrics/collector/persist/PersistenceUtils.java
index 2b7621d..1dc36d6 100644
---
a/src/main/java/org/torproject/metrics/collector/persist/PersistenceUtils.java
+++
b/src/main/java/org/torproject/metrics/collector/persist/PersistenceUtils.java
@@ -20,6 +20,7 @@ import java.nio.file.attribute.BasicFileAttributes;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.util.Date;
+import java.util.stream.Stream;
public class PersistenceUtils {
@@ -132,9 +133,12 @@ public class PersistenceUtils {
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc)
throws IOException {
-if (!pathToClean.equals(dir)
-&& !Files.list(dir).findFirst().isPresent()) {
- Files.delete(dir);
+if (!pathToClean.equals(dir)) {
+ try (Stream files = Files.list(dir)) {
+if (!files.findFirst().isPresent()) {
+ Files.delete(dir);
+}
+ }
}
return FileVisitResult.CONTINUE;
}
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/master] Fix a few paths for cleaning up.
commit 4879cdc219c797e3e0a54226978462fe67d7cf3b
Author: Karsten Loesing
Date: Fri Dec 11 16:55:37 2020 +0100
Fix a few paths for cleaning up.
---
.../metrics/collector/bridgedb/BridgedbMetricsProcessor.java| 6 --
.../metrics/collector/bridgedescs/SanitizedBridgesWriter.java | 6 --
.../metrics/collector/snowflake/SnowflakeStatsDownloader.java | 6 --
3 files changed, 12 insertions(+), 6 deletions(-)
diff --git
a/src/main/java/org/torproject/metrics/collector/bridgedb/BridgedbMetricsProcessor.java
b/src/main/java/org/torproject/metrics/collector/bridgedb/BridgedbMetricsProcessor.java
index d05aa9c..94f697e 100644
---
a/src/main/java/org/torproject/metrics/collector/bridgedb/BridgedbMetricsProcessor.java
+++
b/src/main/java/org/torproject/metrics/collector/bridgedb/BridgedbMetricsProcessor.java
@@ -178,9 +178,11 @@ public class BridgedbMetricsProcessor extends
CollecTorMain {
* in the last three days (seven weeks).
*/
private void cleanUpDirectories() {
-PersistenceUtils.cleanDirectory(Paths.get(this.recentPathName),
+PersistenceUtils.cleanDirectory(
+Paths.get(this.recentPathName).resolve("bridgedb-metrics"),
Instant.now().minus(3, ChronoUnit.DAYS).toEpochMilli());
-PersistenceUtils.cleanDirectory(Paths.get(this.outputPathName),
+PersistenceUtils.cleanDirectory(
+Paths.get(this.outputPathName).resolve("bridgedb-metrics"),
Instant.now().minus(49, ChronoUnit.DAYS).toEpochMilli());
}
}
diff --git
a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
index 7619453..ffafdea 100644
---
a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
+++
b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
@@ -449,9 +449,11 @@ public class SanitizedBridgesWriter extends CollecTorMain {
* in the last three days (seven weeks), and remove the .tmp extension from
* newly written files. */
private void cleanUpDirectories() {
-PersistenceUtils.cleanDirectory(this.recentDirectory,
+PersistenceUtils.cleanDirectory(
+this.recentDirectory.resolve("bridge-descriptors"),
Instant.now().minus(3, ChronoUnit.DAYS).toEpochMilli());
-PersistenceUtils.cleanDirectory(this.outputDirectory,
+PersistenceUtils.cleanDirectory(
+this.outputDirectory.resolve("bridge-descriptors"),
Instant.now().minus(49, ChronoUnit.DAYS).toEpochMilli());
}
}
diff --git
a/src/main/java/org/torproject/metrics/collector/snowflake/SnowflakeStatsDownloader.java
b/src/main/java/org/torproject/metrics/collector/snowflake/SnowflakeStatsDownloader.java
index 93388d5..c9b6b03 100644
---
a/src/main/java/org/torproject/metrics/collector/snowflake/SnowflakeStatsDownloader.java
+++
b/src/main/java/org/torproject/metrics/collector/snowflake/SnowflakeStatsDownloader.java
@@ -156,9 +156,11 @@ public class SnowflakeStatsDownloader extends
CollecTorMain {
/** Delete all files from the rsync (out) directory that have not been
* modified in the last three days (seven weeks). */
private void cleanUpDirectories() {
-PersistenceUtils.cleanDirectory(Paths.get(this.recentPathName),
+PersistenceUtils.cleanDirectory(
+Paths.get(this.recentPathName, "snowflakes"),
Instant.now().minus(3, ChronoUnit.DAYS).toEpochMilli());
-PersistenceUtils.cleanDirectory(Paths.get(this.outputPathName),
+PersistenceUtils.cleanDirectory(
+Paths.get(this.outputPathName, "snowflakes"),
Instant.now().minus(49, ChronoUnit.DAYS).toEpochMilli());
}
}
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/master] Only clean up a single time during sync.
commit 5e5a7d1dfa30ffaf0a27ea4e246de5bef30d1cbc
Author: Karsten Loesing
Date: Fri Dec 11 16:55:58 2020 +0100
Only clean up a single time during sync.
---
src/main/java/org/torproject/metrics/collector/sync/SyncManager.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/src/main/java/org/torproject/metrics/collector/sync/SyncManager.java
b/src/main/java/org/torproject/metrics/collector/sync/SyncManager.java
index 1fa1347..8e9c7d3 100644
--- a/src/main/java/org/torproject/metrics/collector/sync/SyncManager.java
+++ b/src/main/java/org/torproject/metrics/collector/sync/SyncManager.java
@@ -108,11 +108,11 @@ public class SyncManager {
persist.storeDesc(desc, collectionDate.getTime());
}
-persist.cleanDirectory();
descriptorReader.saveHistoryFile(historyFile);
}
logger.info("Done merging {} from {}.", marker, source.getHost());
}
+persist.cleanDirectory();
}
}
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/master] Include microdescriptors when syncing.
commit a1b8ebb9545b148524c6e3db2a1601f228784905
Author: Karsten Loesing
Date: Wed Dec 9 22:01:21 2020 +0100
Include microdescriptors when syncing.
---
CHANGELOG.md | 3 ++
build.xml | 2 +-
.../persist/MicrodescriptorPersistence.java| 36 ++
.../collector/relaydescs/ArchiveWriter.java| 11 +--
.../metrics/collector/sync/SyncPersistence.java| 10 ++
5 files changed, 58 insertions(+), 4 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2928937..22d3517 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,9 @@
- Clean up descriptors written to the `out/` directory by deleting
files that are older than seven weeks.
- Correctly index files that are moved away and back.
+ - Include microdescriptors when syncing from another CollecTor
+instance.
+ - Update to metrics-lib 2.15.0.
# Changes in version 1.16.1 - 2020-08-16
diff --git a/build.xml b/build.xml
index 5dcb29d..f2bda4e 100644
--- a/build.xml
+++ b/build.xml
@@ -12,7 +12,7 @@
-
+
diff --git
a/src/main/java/org/torproject/metrics/collector/persist/MicrodescriptorPersistence.java
b/src/main/java/org/torproject/metrics/collector/persist/MicrodescriptorPersistence.java
new file mode 100644
index 000..41929af
--- /dev/null
+++
b/src/main/java/org/torproject/metrics/collector/persist/MicrodescriptorPersistence.java
@@ -0,0 +1,36 @@
+/* Copyright 2020 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.metrics.collector.persist;
+
+import org.torproject.descriptor.Microdescriptor;
+import org.torproject.metrics.collector.conf.Annotation;
+
+import java.nio.file.Paths;
+
+public class MicrodescriptorPersistence
+extends DescriptorPersistence {
+
+ private static final String RELAY_DESCRIPTORS = "relay-descriptors";
+
+ public MicrodescriptorPersistence(Microdescriptor descriptor, long received,
+ String year, String month) {
+super(descriptor, Annotation.Microdescriptor.bytes());
+calculatePaths(received, year, month);
+ }
+
+ private void calculatePaths(long received, String year, String month) {
+String file = PersistenceUtils.dateTime(received);
+this.recentPath = Paths.get(
+RELAY_DESCRIPTORS, MICRODESCS, "micro",
+file + "-micro-" + year + "-" + month).toString();
+String digest = desc.getDigestSha256Hex();
+this.storagePath = Paths.get(
+RELAY_DESCRIPTORS,
+MICRODESC, year, month, "micro",
+digest.substring(0,1),
+digest.substring(1,2),
+digest).toString();
+ }
+}
+
diff --git
a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
index 28472f8..5c58f23 100644
---
a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
+++
b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
@@ -7,6 +7,7 @@ import org.torproject.descriptor.BandwidthFile;
import org.torproject.descriptor.Descriptor;
import org.torproject.descriptor.DescriptorParser;
import org.torproject.descriptor.DescriptorSourceFactory;
+import org.torproject.descriptor.Microdescriptor;
import org.torproject.descriptor.RelayExtraInfoDescriptor;
import org.torproject.descriptor.RelayNetworkStatusConsensus;
import org.torproject.descriptor.RelayNetworkStatusVote;
@@ -109,6 +110,8 @@ public class ArchiveWriter extends CollecTorMain {
this.mapPathDescriptors.put(
"recent/relay-descriptors/microdescs/consensus-microdesc",
RelayNetworkStatusConsensus.class);
+this.mapPathDescriptors.put("recent/relay-descriptors/microdescs/micro/",
+Microdescriptor.class);
this.mapPathDescriptors.put("recent/relay-descriptors/server-descriptors",
RelayServerDescriptor.class);
this.mapPathDescriptors.put("recent/relay-descriptors/extra-infos",
@@ -801,15 +804,17 @@ public class ArchiveWriter extends CollecTorMain {
* file written in the first call. However, this method must be
* called twice to store the same microdescriptor in two different
* valid-after months. */
-SimpleDateFormat descriptorFormat = new SimpleDateFormat("/MM/");
+SimpleDateFormat descriptorFormat = new SimpleDateFormat("-MM");
+String[] yearMonth = descriptorFormat.format(validAfter).split("-");
File tarballFile = Paths.get(this.outputDirectory, MICRODESC,
-descriptorFormat.format(validAfter), MICRO,
+yearMonth[0], yearMonth[1], MICRO,
microdescriptorDigest.substring(0, 1),
microdescriptorDigest.substring(1, 2),
microdescriptorDigest).toFile();
boolean tarballFileExistedBefore = tarballFile.exists(
[tor-commits] [collector/master] Add log statement for current memory usage.
commit bddc73ce6700a1498cbffb294de135949d02a41d
Author: Karsten Loesing
Date: Thu Dec 10 22:08:42 2020 +0100
Add log statement for current memory usage.
---
.../org/torproject/metrics/collector/indexer/CreateIndexJson.java | 4
1 file changed, 4 insertions(+)
diff --git
a/src/main/java/org/torproject/metrics/collector/indexer/CreateIndexJson.java
b/src/main/java/org/torproject/metrics/collector/indexer/CreateIndexJson.java
index 3d86947..887a3e1 100644
---
a/src/main/java/org/torproject/metrics/collector/indexer/CreateIndexJson.java
+++
b/src/main/java/org/torproject/metrics/collector/indexer/CreateIndexJson.java
@@ -245,6 +245,10 @@ public class CreateIndexJson extends CollecTorMain {
logger.info("Writing uncompressed and compressed index.json files to "
+ "disk.");
this.writeIndex(this.index, now);
+ Runtime rt = Runtime.getRuntime();
+ logger.info("Current memory usage is: free = {} B, total = {} B, "
+ + "max = {} B.",
+ rt.freeMemory(), rt.totalMemory(), rt.maxMemory());
logger.info("Pausing until next index update run.");
} catch (IOException e) {
logger.error("I/O error while updating index.json files. Trying again in
"
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/develop] Correct issue number in change log.
commit 15ff2beed583768964c6c3e19ea48f5d4319286b Author: Karsten Loesing Date: Sun Dec 13 22:58:39 2020 +0100 Correct issue number in change log. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2655e01..5bce6f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Changes in version 0.9 - 2020-1?-?? - Avoid tracebacks when visualizing analysis files containing only - unsuccessful measurements. Fixes #44012. + unsuccessful measurements. Fixes #40012. # Changes in version 0.8 - 2020-09-16 ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/develop] Avoid tracebacks when visualizing measurements.
commit fe6ff0874dfd9b18bf5eb71c2394431ca44874bc
Author: Karsten Loesing
Date: Tue Nov 24 12:17:21 2020 +0100
Avoid tracebacks when visualizing measurements.
Attempting to visualize analysis files containing only unsuccessful
measurements results in various tracebacks.
With this patch we check more carefully whether a data frame is empty
before adding a plot.
Another related change is that we always include
"time_to_{first,last}_byte" and "mbps" columns in the CSV output,
regardless of whether there are there are any non-null values in the
data. See also #40004 for a previous related change.
And we check whether a Tor stream identifier exists before retrieving
a Tor stream.
Finally, we only include TTFB/TTLB if the usecs value is non-zero.
Fixes #44012.
---
CHANGELOG.md | 5 +
onionperf/visualization.py | 37 +
2 files changed, 30 insertions(+), 12 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2c1f5ca..2655e01 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,8 @@
+# Changes in version 0.9 - 2020-1?-??
+
+ - Avoid tracebacks when visualizing analysis files containing only
+ unsuccessful measurements. Fixes #44012.
+
# Changes in version 0.8 - 2020-09-16
- Add a new `onionperf filter` mode that takes an OnionPerf analysis
diff --git a/onionperf/visualization.py b/onionperf/visualization.py
index 2cd3161..75ec2b5 100644
--- a/onionperf/visualization.py
+++ b/onionperf/visualization.py
@@ -66,6 +66,7 @@ class TGenVisualization(Visualization):
tgen_streams = analysis.get_tgen_streams(client)
tgen_transfers = analysis.get_tgen_transfers(client)
while tgen_streams or tgen_transfers:
+stream = {"time_to_first_byte": None,
"time_to_last_byte": None, "error_code": None, "mbps": None}
error_code = None
source_port = None
unix_ts_end = None
@@ -76,15 +77,15 @@ class TGenVisualization(Visualization):
# that value with unit megabits per second.
if tgen_streams:
stream_id, stream_data = tgen_streams.popitem()
-stream = {"id": stream_id, "label": label,
- "filesize_bytes":
int(stream_data["stream_info"]["recvsize"]),
- "error_code": None}
+stream["id"] = stream_id
+stream["label"] = label
+stream["filesize_bytes"] =
int(stream_data["stream_info"]["recvsize"])
stream["server"] = "onion" if ".onion:" in
stream_data["transport_info"]["remote"] else "public"
if "time_info" in stream_data:
s = stream_data["time_info"]
-if "usecs-to-first-byte-recv" in s:
+if "usecs-to-first-byte-recv" in s and
float(s["usecs-to-first-byte-recv"]) >= 0:
stream["time_to_first_byte"] =
float(s["usecs-to-first-byte-recv"])/100
-if "usecs-to-last-byte-recv" in s:
+if "usecs-to-last-byte-recv" in s and
float(s["usecs-to-last-byte-recv"]) >= 0:
stream["time_to_last_byte"] =
float(s["usecs-to-last-byte-recv"])/100
if "elapsed_seconds" in stream_data:
s = stream_data["elapsed_seconds"]
@@ -100,9 +101,9 @@ class TGenVisualization(Visualization):
stream["start"] =
datetime.datetime.utcfromtimestamp(stream_data["unix_ts_start"])
elif tgen_transfers:
transfer_id, transfer_data =
tgen_transfers.popitem()
-stream = {"id": transfer_id, "label": label,
- "filesize_bytes":
transfer_data["filesize_bytes"],
- "error_code": None}
+stream["id"] = transfer_id
+stream["label"] = label
+stream["filesize_bytes"] =
transfer_data["filesize_bytes
[tor-commits] [onionperf/develop] Merge branch 'task-40012' into develop
commit f8944174633f3b0fa5e74747086c02761f83ee70 Merge: ce36170 c46deb0 Author: Karsten Loesing Date: Sun Dec 13 22:57:46 2020 +0100 Merge branch 'task-40012' into develop CHANGELOG.md | 5 + onionperf/visualization.py | 42 -- 2 files changed, 33 insertions(+), 14 deletions(-) ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/develop] Move two data.empty checks after data.dropna calls.
commit c46deb00967899bdd9c15b9b7ee41ca5c53b
Author: Karsten Loesing
Date: Sun Dec 13 22:26:52 2020 +0100
Move two data.empty checks after data.dropna calls.
Suggested by acute, still related to #40012.
---
onionperf/visualization.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/onionperf/visualization.py b/onionperf/visualization.py
index 75ec2b5..a793791 100644
--- a/onionperf/visualization.py
+++ b/onionperf/visualization.py
@@ -298,25 +298,26 @@ class TGenVisualization(Visualization):
plt.close()
def __draw_countplot(self, x, data, title, xlabel, ylabel, hue=None,
hue_name=None):
+data = data.dropna(subset=[x])
if data.empty:
return
plt.figure()
if hue is not None:
data = data.rename(columns={hue: hue_name})
-g = sns.countplot(data=data.dropna(subset=[x]), x=x, hue=hue_name)
+g = sns.countplot(data=data, x=x, hue=hue_name)
g.set(xlabel=xlabel, ylabel=ylabel, title=title)
sns.despine()
self.page.savefig()
plt.close()
def __draw_stripplot(self, x, y, hue, hue_name, data, title, xlabel,
ylabel):
+data = data.dropna(subset=[y])
if data.empty:
return
plt.figure()
data = data.rename(columns={hue: hue_name})
xmin = data[x].min()
xmax = data[x].max()
-data = data.dropna(subset=[y])
g = sns.stripplot(data=data, x=x, y=y, hue=hue_name)
g.set(title=title, xlabel=xlabel, ylabel=ylabel,
xlim=(xmin - 0.03 * (xmax - xmin), xmax + 0.03 * (xmax - xmin)))
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/develop] Merge branch 'task-40008' into develop
commit ce36170f03b540fc4e2c8716a0bddade37196162 Merge: 118d149 0bf86a6 Author: Karsten Loesing Date: Sun Dec 13 22:01:45 2020 +0100 Merge branch 'task-40008' into develop onionperf/onionperf | 3 +++ 1 file changed, 3 insertions(+) ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/develop] Log version number in each execution.
commit 0bf86a60854b567fd792728e1957dcf5107d1229
Author: Karsten Loesing
Date: Sun Dec 13 22:01:09 2020 +0100
Log version number in each execution.
Implements #40008.
---
onionperf/onionperf | 3 +++
1 file changed, 3 insertions(+)
diff --git a/onionperf/onionperf b/onionperf/onionperf
index 032cd7d..90b5bf6 100755
--- a/onionperf/onionperf
+++ b/onionperf/onionperf
@@ -14,6 +14,8 @@ from socket import gethostname
import onionperf.util as util
from onionperf.monitor import get_supported_torctl_events
+__version__ = '0.8'
+
DESC_MAIN = """
OnionPerf is a utility to monitor, measure, analyze, and visualize the
performance of Tor and Onion Services.
@@ -379,6 +381,7 @@ files generated by this script will be written""",
sys.exit(1)
else:
args = main_parser.parse_args()
+logging.info("Starting OnionPerf version {0} in {1}
mode.".format(__version__, sys.argv[1]))
args.func(args)
def monitor(args):
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-lib/master] Optimize parsing large files with many descriptors.
commit ff7e36c15626bdc24df54ebd94da5ab58f4de4c4
Author: Karsten Loesing
Date: Thu Dec 10 17:54:02 2020 +0100
Optimize parsing large files with many descriptors.
When parsing a large file with many descriptors we would repeatedly
search the remaining file for the sequence "newline + keyword + space"
and then "newline + keyword + newline" to find the start of the next
descriptor. However, if the keyword is always followed by newline, the
first search would always fail.
The optimization here is to search once whether the keyword is
followed by space or newline and avoid unnecessary searches when going
through the file.
In the long term we should use a better parser. But in the short term
this optimization will have a major impact on performance, in
particular with regard to concatenated microdescriptors.
---
CHANGELOG.md | 3 +++
.../descriptor/impl/DescriptorParserImpl.java | 27 ++
2 files changed, 21 insertions(+), 9 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8ff5723..828718d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,9 @@
- Parse version 3 onion service statistics contained in extra-info
descriptors.
+ * Medium changes
+ - Optimize parsing of large files containing many descriptors.
+
# Changes in version 2.14.0 - 2020-08-07
diff --git
a/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java
b/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java
index e008e7a..abe4411 100644
--- a/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java
@@ -181,16 +181,25 @@ public class DescriptorParserImpl implements
DescriptorParser {
String ascii = new String(rawDescriptorBytes, StandardCharsets.US_ASCII);
boolean containsAnnotations = ascii.startsWith("@")
|| ascii.contains(NL + "@");
+boolean containsKeywordSpace = ascii.startsWith(key.keyword + SP)
+|| ascii.contains(NL + key.keyword + SP);
+boolean containsKeywordNewline = ascii.startsWith(key.keyword + NL)
+|| ascii.contains(NL + key.keyword + NL);
while (startAnnotations < endAllDescriptors) {
- int startDescriptor;
- if (startAnnotations == ascii.indexOf(key.keyword + SP,
- startAnnotations) || startAnnotations == ascii.indexOf(
- key.keyword + NL)) {
+ int startDescriptor = -1;
+ if ((containsKeywordSpace
+ && startAnnotations == ascii.indexOf(key.keyword + SP,
+ startAnnotations))
+ || (containsKeywordNewline
+ && startAnnotations == ascii.indexOf(key.keyword + NL,
+ startAnnotations))) {
startDescriptor = startAnnotations;
} else {
-startDescriptor = ascii.indexOf(NL + key.keyword + SP,
-startAnnotations - 1);
-if (startDescriptor < 0) {
+if (containsKeywordSpace) {
+ startDescriptor = ascii.indexOf(NL + key.keyword + SP,
+ startAnnotations - 1);
+}
+if (startDescriptor < 0 && containsKeywordNewline) {
startDescriptor = ascii.indexOf(NL + key.keyword + NL,
startAnnotations - 1);
}
@@ -204,10 +213,10 @@ public class DescriptorParserImpl implements
DescriptorParser {
if (containsAnnotations) {
endDescriptor = ascii.indexOf(NL + "@", startDescriptor);
}
- if (endDescriptor < 0) {
+ if (endDescriptor < 0 && containsKeywordSpace) {
endDescriptor = ascii.indexOf(NL + key.keyword + SP, startDescriptor);
}
- if (endDescriptor < 0) {
+ if (endDescriptor < 0 && containsKeywordNewline) {
endDescriptor = ascii.indexOf(NL + key.keyword + NL, startDescriptor);
}
if (endDescriptor < 0) {
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-lib/master] Prepare for 2.15.0 release.
commit 07cab7f5604fe943c915c91251b8da322d53036c Author: Karsten Loesing Date: Fri Dec 11 14:50:18 2020 +0100 Prepare for 2.15.0 release. --- CHANGELOG.md | 4 +--- build.xml| 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bee22c3..cb5bce6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,8 @@ -# Changes in version 2.15.0 - 2020-??-?? +# Changes in version 2.15.0 - 2020-12-11 * Medium changes - Parse version 3 onion service statistics contained in extra-info descriptors. - - * Medium changes - Optimize parsing of large files containing many descriptors. * Minor changes diff --git a/build.xml b/build.xml index 72990c6..1a9e555 100644 --- a/build.xml +++ b/build.xml @@ -7,7 +7,7 @@ - + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-lib/master] Parse version 3 onion service statistics lines.
commit ee7c3eb73824a84e33b6c44b60fa8a32f37ab71e
Author: Karsten Loesing
Date: Tue Dec 8 10:40:19 2020 +0100
Parse version 3 onion service statistics lines.
Implements the library part of tpo/metrics/statistics#40002.
---
CHANGELOG.md | 6 +-
.../torproject/descriptor/ExtraInfoDescriptor.java | 55 +
.../descriptor/impl/ExtraInfoDescriptorImpl.java | 93 ++
.../java/org/torproject/descriptor/impl/Key.java | 3 +
.../impl/ExtraInfoDescriptorImplTest.java | 46 +++
5 files changed, 202 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c6886e8..8ff5723 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,8 @@
-# Changes in version 2.??.? - 2020-??-??
+# Changes in version 2.15.0 - 2020-??-??
+
+ * Medium changes
+ - Parse version 3 onion service statistics contained in extra-info
+ descriptors.
# Changes in version 2.14.0 - 2020-08-07
diff --git a/src/main/java/org/torproject/descriptor/ExtraInfoDescriptor.java
b/src/main/java/org/torproject/descriptor/ExtraInfoDescriptor.java
index e094fed..b553a22 100644
--- a/src/main/java/org/torproject/descriptor/ExtraInfoDescriptor.java
+++ b/src/main/java/org/torproject/descriptor/ExtraInfoDescriptor.java
@@ -674,6 +674,61 @@ public interface ExtraInfoDescriptor extends Descriptor {
*/
Map getHidservDirOnionsSeenParameters();
+ /**
+ * Return the time in milliseconds since the epoch when the included version
3
+ * onion service statistics interval ended, or -1 if no such statistics are
+ * included.
+ *
+ * @since 2.15.0
+ */
+ long getHidservV3StatsEndMillis();
+
+ /**
+ * Return the interval length of the included version 3 onion service
+ * statistics in seconds, or -1 if no such statistics are included.
+ *
+ * @since 2.15.0
+ */
+ long getHidservV3StatsIntervalLength();
+
+ /**
+ * Return the approximate number of RELAY cells seen in either direction on a
+ * version 3 onion service circuit after receiving and successfully
processing
+ * a RENDEZVOUS1 cell, or null if no such statistics are included.
+ *
+ * @since 2.15.0
+ */
+ Double getHidservRendV3RelayedCells();
+
+ /**
+ * Return the obfuscation parameters applied to the original measurement
value
+ * of RELAY cells seen in either direction on a version 3 onion service
+ * circuit after receiving and successfully processing a RENDEZVOUS1 cell, or
+ * null if no such statistics are included.
+ *
+ * @since 2.15.0
+ */
+ Map getHidservRendV3RelayedCellsParameters();
+
+ /**
+ * Return the approximate number of unique version 3 onion service identities
+ * seen in descriptors published to and accepted by this onion service
+ * directory, or null if no such statistics are included.
+ *
+ * @since 2.15.0
+ */
+ Double getHidservDirV3OnionsSeen();
+
+ /**
+ * Return the obfuscation parameters applied to the original measurement
value
+ * of unique version 3 onion service identities seen in descriptors published
+ * to and accepted by this onion service directory, or null if no such
+ * statistics are included.
+ *
+ * @since 2.15.0
+ */
+ Map getHidservDirV3OnionsSeenParameters();
+
/**
* Return the time in milliseconds since the epoch when the included
* padding-counts statistics ended, or -1 if no such statistics are included.
diff --git
a/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java
b/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java
index 5cab6ab..4f87237 100644
--- a/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java
@@ -225,6 +225,15 @@ public abstract class ExtraInfoDescriptorImpl extends
DescriptorImpl
case HIDSERV_DIR_ONIONS_SEEN:
this.parseHidservDirOnionsSeenLine(line, partsNoOpt);
break;
+case HIDSERV_V3_STATS_END:
+ this.parseHidservV3StatsEndLine(line, partsNoOpt);
+ break;
+case HIDSERV_REND_V3_RELAYED_CELLS:
+ this.parseHidservRendV3RelayedCellsLine(line, partsNoOpt);
+ break;
+case HIDSERV_DIR_V3_ONIONS_SEEN:
+ this.parseHidservDirV3OnionsSeenLine(line, partsNoOpt);
+ break;
case PADDING_COUNTS:
this.parsePaddingCountsLine(line, partsNoOpt);
break;
@@ -764,6 +773,46 @@ public abstract class ExtraInfoDescriptorImpl extends
DescriptorImpl
partsNoOpt, 2);
}
+ private void parseHidservV3StatsEndLine(String line,
+ String[] partsNoOpt) throws DescriptorParseException {
+long[] parsedStatsEndData = this.parseStatsEndLine(line, partsNoOpt,
+5);
+this.hidservV3StatsEndMillis = parsedStatsEndData[0];
+this.hidservV3StatsIntervalLength = parsedStatsEndData[1];
+ }
+
+ private void
[tor-commits] [metrics-lib/master] Bump version to 2.15.0-dev.
commit 179d47ee259ffe86302e57dad6cbc39fdbb72f0b Author: Karsten Loesing Date: Fri Dec 11 14:53:56 2020 +0100 Bump version to 2.15.0-dev. --- CHANGELOG.md | 3 +++ build.xml| 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb5bce6..ac865a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +# Changes in version 2.??.? - 2020-??-?? + + # Changes in version 2.15.0 - 2020-12-11 * Medium changes diff --git a/build.xml b/build.xml index 1a9e555..7544a32 100644 --- a/build.xml +++ b/build.xml @@ -7,7 +7,7 @@ - + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-lib/master] Provide microdescriptor digest in hex encoding.
commit 664921eb50491a774a5281bf1c12836ab7dedd94
Author: Karsten Loesing
Date: Fri Dec 11 10:22:39 2020 +0100
Provide microdescriptor digest in hex encoding.
---
CHANGELOG.md | 3 +++
.../java/org/torproject/descriptor/Microdescriptor.java | 9 +
.../torproject/descriptor/impl/MicrodescriptorImpl.java | 16
.../descriptor/impl/MicrodescriptorImplTest.java | 3 +++
4 files changed, 31 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 828718d..bee22c3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,9 @@
* Medium changes
- Optimize parsing of large files containing many descriptors.
+ * Minor changes
+ - Provide microdescriptor SHA-256 digest in hexadecimal encoding.
+
# Changes in version 2.14.0 - 2020-08-07
diff --git a/src/main/java/org/torproject/descriptor/Microdescriptor.java
b/src/main/java/org/torproject/descriptor/Microdescriptor.java
index feaf00b..0d329b7 100644
--- a/src/main/java/org/torproject/descriptor/Microdescriptor.java
+++ b/src/main/java/org/torproject/descriptor/Microdescriptor.java
@@ -32,6 +32,15 @@ public interface Microdescriptor extends Descriptor {
*/
String getDigestSha256Base64();
+ /**
+ * Return the SHA-256 descriptor digest, encoded as 64 lower-case hexadecimal
+ * characters, that can be used as file name when writing this descriptor to
+ * disk.
+ *
+ * @since 2.15.0
+ */
+ String getDigestSha256Hex();
+
/**
* Return the RSA-1024 public key in PEM format used to encrypt CREATE
* cells for this server, or null if the descriptor doesn't contain an
diff --git
a/src/main/java/org/torproject/descriptor/impl/MicrodescriptorImpl.java
b/src/main/java/org/torproject/descriptor/impl/MicrodescriptorImpl.java
index 87ab7ae..8d4ac1b 100644
--- a/src/main/java/org/torproject/descriptor/impl/MicrodescriptorImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/MicrodescriptorImpl.java
@@ -6,6 +6,9 @@ package org.torproject.descriptor.impl;
import org.torproject.descriptor.DescriptorParseException;
import org.torproject.descriptor.Microdescriptor;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.codec.binary.Hex;
+
import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
@@ -26,6 +29,7 @@ public class MicrodescriptorImpl extends DescriptorImpl
super(descriptorBytes, offsetAndLength, descriptorFile, false);
this.parseDescriptorBytes();
this.calculateDigestSha256Base64(Key.ONION_KEY.keyword + NL);
+this.convertDigestSha256Base64ToHex();
this.checkExactlyOnceKeys(EnumSet.of(Key.ONION_KEY));
Set atMostOnceKeys = EnumSet.of(
Key.NTOR_ONION_KEY, Key.FAMILY, Key.P, Key.P6, Key.ID);
@@ -212,6 +216,18 @@ public class MicrodescriptorImpl extends DescriptorImpl
}
}
+ private void convertDigestSha256Base64ToHex() {
+this.digestSha256Hex = Hex.encodeHexString(Base64.decodeBase64(
+this.getDigestSha256Base64()));
+ }
+
+ private String digestSha256Hex;
+
+ @Override
+ public String getDigestSha256Hex() {
+return this.digestSha256Hex;
+ }
+
private String onionKey;
@Override
diff --git
a/src/test/java/org/torproject/descriptor/impl/MicrodescriptorImplTest.java
b/src/test/java/org/torproject/descriptor/impl/MicrodescriptorImplTest.java
index 128d39a..fbc2fc9 100644
--- a/src/test/java/org/torproject/descriptor/impl/MicrodescriptorImplTest.java
+++ b/src/test/java/org/torproject/descriptor/impl/MicrodescriptorImplTest.java
@@ -74,6 +74,9 @@ public class MicrodescriptorImplTest {
Microdescriptor micro = DescriptorBuilder.createWithDefaultLines();
assertEquals("ER1AC4KqT//o3pJDrqlmej5G2qW1EQYEr/IrMQHNc6I",
micro.getDigestSha256Base64());
+assertEquals(
+"111d400b82aa4fffe8de9243aea9667a3e46daa5b5110604aff22b3101cd73a2",
+micro.getDigestSha256Hex());
}
@Test
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/master] Use persist package for writing bridge descriptors.
commit 7439fa933c9b686a8650d0b9d5c398fdd434f749
Author: Karsten Loesing
Date: Tue Dec 1 22:32:46 2020 +0100
Use persist package for writing bridge descriptors.
Implements #25307.
---
.../bridgedescs/SanitizedBridgesWriter.java| 104 +++--
.../BridgeExtraInfoDescriptorPersistence.java | 54 +++
.../persist/BridgeExtraInfoPersistence.java| 39
.../persist/BridgeNetworkStatusPersistence.java| 56 +++
.../persist/BridgeServerDescriptorPersistence.java | 43 ++---
.../collector/persist/DescriptorPersistence.java | 40 +---
.../collector/persist/StatusPersistence.java | 41
.../metrics/collector/sync/SyncPersistence.java| 10 +-
.../bridgedescs/SanitizedBridgesWriterTest.java| 2 +-
9 files changed, 188 insertions(+), 201 deletions(-)
diff --git
a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
index 5e24f5d..7619453 100644
---
a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
+++
b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
@@ -10,6 +10,9 @@ import org.torproject.metrics.collector.conf.Configuration;
import org.torproject.metrics.collector.conf.ConfigurationException;
import org.torproject.metrics.collector.conf.Key;
import org.torproject.metrics.collector.cron.CollecTorMain;
+import
org.torproject.metrics.collector.persist.BridgeExtraInfoDescriptorPersistence;
+import org.torproject.metrics.collector.persist.BridgeNetworkStatusPersistence;
+import
org.torproject.metrics.collector.persist.BridgeServerDescriptorPersistence;
import org.torproject.metrics.collector.persist.PersistenceUtils;
import org.apache.commons.codec.binary.Hex;
@@ -28,8 +31,6 @@ import java.io.StringReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.StandardOpenOption;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
@@ -57,7 +58,6 @@ public class SanitizedBridgesWriter extends CollecTorMain {
private static final Logger logger = LoggerFactory.getLogger(
SanitizedBridgesWriter.class);
- private static final String BRIDGE_DESCRIPTORS = "bridge-descriptors";
/** Initialize configuration. */
public SanitizedBridgesWriter(Configuration config) {
@@ -91,10 +91,8 @@ public class SanitizedBridgesWriter extends CollecTorMain {
@Override
protected void startProcessing() throws ConfigurationException {
-this.outputDirectory = config.getPath(Key.OutputPath)
-.resolve(BRIDGE_DESCRIPTORS);
-this.recentDirectory = config.getPath(Key.RecentPath)
-.resolve(BRIDGE_DESCRIPTORS);
+this.outputDirectory = config.getPath(Key.OutputPath);
+this.recentDirectory = config.getPath(Key.RecentPath);
Path inputDirectory = config.getPath(Key.BridgeLocalOrigins);
Path statsDirectory = config.getPath(Key.StatsPath);
boolean replaceIpAddressesWithHashes =
@@ -352,27 +350,9 @@ public class SanitizedBridgesWriter extends CollecTorMain {
|| publicationTime.compareTo(maxNetworkStatusPublishedTime) > 0) {
maxNetworkStatusPublishedTime = publicationTime;
}
-try {
- String syear = publicationTime.substring(0, 4);
- String smonth = publicationTime.substring(5, 7);
- String sday = publicationTime.substring(8, 10);
- String stime = publicationTime.substring(11, 13)
- + publicationTime.substring(14, 16)
- + publicationTime.substring(17, 19);
- String fileName = syear + smonth + sday + "-" + stime + "-"
- + authorityFingerprint;
- Path tarballFile = this.outputDirectory.resolve(
- Paths.get(syear, smonth, "statuses", sday, fileName));
- Path rsyncFile = this.recentDirectory.resolve(
- Paths.get("statuses", fileName));
- for (Path outputFile : new Path[] { tarballFile, rsyncFile }) {
-Files.createDirectories(outputFile.getParent());
-Files.write(outputFile, scrubbedBytes);
- }
-} catch (IOException e) {
- logger.warn("Could not write sanitized bridge "
- + "network status to disk.", e);
-}
+new BridgeNetworkStatusPersistence(scrubbedBytes, publicationTime,
+authorityFingerprint)
+.storeAll(this.recentDirectory, this.outputDirectory);
}
private String maxServerDescriptorPublishedTime = null;
@@ -398,36 +378,9 @@ public class SanitizedBridgesWriter extends CollecTorMain {
}
String descriptorDigest
= sanitizedBridgeServerDescriptor.getDescriptorDigest();
-
-/* Determine filename of sanitized server descriptor. */
-String dyear =
[tor-commits] [collector/master] Use java8 datetime classes in bridgedesc module.
commit 27a21c0b37d504b4ab3eeb5b206b5590fc82d396
Author: Karsten Loesing
Date: Tue Dec 1 11:15:21 2020 +0100
Use java8 datetime classes in bridgedesc module.
Implements #25309.
---
.../bridgedescs/SanitizedBridgeNetworkStatus.java | 31 ++-
.../bridgedescs/SanitizedBridgesWriter.java| 64 +++---
.../BridgePoolAssignmentsProcessor.java| 8 +--
3 files changed, 51 insertions(+), 52 deletions(-)
diff --git
a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgeNetworkStatus.java
b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgeNetworkStatus.java
index d94cb0d..8087e84 100644
---
a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgeNetworkStatus.java
+++
b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgeNetworkStatus.java
@@ -15,8 +15,8 @@ import java.io.BufferedReader;
import java.io.IOException;
import java.io.StringReader;
import java.nio.charset.StandardCharsets;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
import java.util.SortedMap;
import java.util.TreeMap;
@@ -186,24 +186,25 @@ public class SanitizedBridgeNetworkStatus extends
SanitizedBridgeDescriptor {
/* Check if we can tell from the descriptor publication times
* whether this status is possibly stale. */
- SimpleDateFormat formatter = new SimpleDateFormat(
- "-MM-dd HH:mm:ss");
+ DateTimeFormatter formatter = DateTimeFormatter.ofPattern(
+ "-MM-dd HH:mm:ss");
if (null == mostRecentDescPublished) {
logger.warn("The bridge network status published at {}"
+ " does not contain a single entry. Please ask the bridge "
+ "authority operator to check!", this.publishedString);
- } else if (formatter.parse(this.publishedString).getTime()
- - formatter.parse(mostRecentDescPublished).getTime()
- > 60L * 60L * 1000L) {
-logger.warn("The most recent descriptor in the bridge "
-+ "network status published at {} was published at {} which is
"
-+ "more than 1 hour before the status. This is a sign for "
-+ "the status being stale. Please check!",
-this.publishedString, mostRecentDescPublished);
+ } else {
+LocalDateTime networkStatusTime
+= LocalDateTime.parse(this.publishedString, formatter);
+LocalDateTime mostRecentDescTime
+= LocalDateTime.parse(mostRecentDescPublished, formatter);
+if (mostRecentDescTime.isBefore(networkStatusTime.minusHours(1L))) {
+ logger.warn("The most recent descriptor in the bridge "
+ + "network status published at {} was published at {} which is "
+ + "more than 1 hour before the status. This is a sign for "
+ + "the status being stale. Please check!",
+ this.publishedString, mostRecentDescPublished);
+}
}
-} catch (ParseException e) {
- logger.warn("Could not parse timestamp in bridge network status.", e);
- return false;
} catch (IOException e) {
logger.warn("Could not parse bridge network status.", e);
return false;
diff --git
a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
index d5009e1..5e24f5d 100644
---
a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
+++
b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
@@ -30,9 +30,9 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.util.HashSet;
import java.util.Set;
@@ -99,10 +99,9 @@ public class SanitizedBridgesWriter extends CollecTorMain {
Path statsDirectory = config.getPath(Key.StatsPath);
boolean replaceIpAddressesWithHashes =
config.getBool(Key.ReplaceIpAddressesWithHashes);
-SimpleDateFormat rsyncCatFormat = new SimpleDateFormat(
-"-MM-dd-HH-mm-ss");
-this.rsyncCatString = rsyncCatFormat.format(
-System.currentTimeMillis());
+DateTimeFormatter rsyncCatFormat = DateTimeFormatter.ofPattern(
+"-MM-dd-HH-mm-ss");
+this.rsyncCatString = LocalDateTime.now().format(rsyncCatFormat);
Path bridgeIpSecr
[tor-commits] [collector/master] Move sanitizing code to one class per type.
commit 2e8cdf7fe1cd11b6afe599512e4844c4234e257a
Author: Karsten Loesing
Date: Tue Dec 1 10:35:26 2020 +0100
Move sanitizing code to one class per type.
Part of #20542.
---
.../bridgedescs/SanitizedBridgeDescriptor.java | 118
.../SanitizedBridgeExtraInfoDescriptor.java| 192 +
.../bridgedescs/SanitizedBridgeNetworkStatus.java | 230 ++
.../SanitizedBridgeServerDescriptor.java | 360 ++
.../bridgedescs/SanitizedBridgesWriter.java| 771 +
5 files changed, 934 insertions(+), 737 deletions(-)
diff --git
a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgeDescriptor.java
b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgeDescriptor.java
new file mode 100644
index 000..5ddeefe
--- /dev/null
+++
b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgeDescriptor.java
@@ -0,0 +1,118 @@
+/* Copyright 2010--2020 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.metrics.collector.bridgedescs;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.charset.StandardCharsets;
+
+public abstract class SanitizedBridgeDescriptor {
+
+ private static final Logger logger = LoggerFactory.getLogger(
+ SanitizedBridgeDescriptor.class);
+
+ protected byte[] originalBytes;
+
+ protected SensitivePartsSanitizer sensitivePartsSanitizer;
+
+ protected byte[] sanitizedBytes;
+
+ protected String publishedString;
+
+ SanitizedBridgeDescriptor(byte[] originalBytes,
+ SensitivePartsSanitizer sensitivePartsSanitizer) {
+this.originalBytes = originalBytes;
+this.sensitivePartsSanitizer = sensitivePartsSanitizer;
+ }
+
+ protected String parseMasterKeyEd25519FromIdentityEd25519(
+ String identityEd25519Base64) {
+byte[] identityEd25519 = Base64.decodeBase64(identityEd25519Base64);
+if (identityEd25519.length < 40) {
+ logger.warn("Invalid length of identity-ed25519 (in bytes): {}",
+ identityEd25519.length);
+} else if (identityEd25519[0] != 0x01) {
+ logger.warn("Unknown version in identity-ed25519: {}",
+ identityEd25519[0]);
+} else if (identityEd25519[1] != 0x04) {
+ logger.warn("Unknown cert type in identity-ed25519: {}",
+ identityEd25519[1]);
+} else if (identityEd25519[6] != 0x01) {
+ logger.warn("Unknown certified key type in identity-ed25519: {}",
+ identityEd25519[1]);
+} else if (identityEd25519[39] == 0x00) {
+ logger.warn("No extensions in identity-ed25519 (which "
+ + "would contain the encoded master-key-ed25519): {}",
+ identityEd25519[39]);
+} else {
+ int extensionStart = 40;
+ for (int i = 0; i < (int) identityEd25519[39]; i++) {
+if (identityEd25519.length < extensionStart + 4) {
+ logger.warn("Invalid extension with id {} in identity-ed25519.", i);
+ break;
+}
+int extensionLength = identityEd25519[extensionStart];
+extensionLength <<= 8;
+extensionLength += identityEd25519[extensionStart + 1];
+int extensionType = identityEd25519[extensionStart + 2];
+if (extensionLength == 32 && extensionType == 4) {
+ if (identityEd25519.length < extensionStart + 4 + 32) {
+logger.warn("Invalid extension with id {} in identity-ed25519.",
i);
+break;
+ }
+ byte[] masterKeyEd25519 = new byte[32];
+ System.arraycopy(identityEd25519, extensionStart + 4,
+ masterKeyEd25519, 0, masterKeyEd25519.length);
+ String masterKeyEd25519Base64 = Base64.encodeBase64String(
+ masterKeyEd25519);
+ return masterKeyEd25519Base64.replaceAll("=", "");
+}
+extensionStart += 4 + extensionLength;
+ }
+}
+logger.warn("Unable to locate master-key-ed25519 in identity-ed25519.");
+return null;
+ }
+
+ protected String computeDescriptorDigest(byte[] descriptorBytes,
+ String startToken, String sigToken) {
+String descriptorDigest = null;
+String ascii = new String(descriptorBytes, StandardCharsets.US_ASCII);
+int start = ascii.indexOf(startToken);
+int sig = ascii.indexOf(sigToken) + sigToken.length();
+if (start >= 0 && sig >= 0 && sig > start) {
+ byte[] forDigest = new byte[sig - start];
+ System.arraycopy(descriptorBytes, start, forDigest, 0, sig - start);
+ descriptorDigest = DigestUtils.sha1Hex(DigestUtils.sha1(forDigest));
+}
+if (descriptorDigest == null) {
+ logger.warn("Could not calculate extra-info descriptor digest.");
[tor-commits] [collector/master] Move lower-level sanitizing code to its own class.
commit a2fdbf3c6f67e5ddb735773e1ab456ee4f464555
Author: Karsten Loesing
Date: Mon Nov 30 21:59:17 2020 +0100
Move lower-level sanitizing code to its own class.
Part of #20542.
---
.../bridgedescs/SanitizedBridgesWriter.java| 404 ++---
.../bridgedescs/SensitivePartsSanitizer.java | 378 +++
.../bridgedescs/SanitizedBridgesWriterTest.java| 2 +
3 files changed, 410 insertions(+), 374 deletions(-)
diff --git
a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
index 34156c2..843aa40 100644
---
a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
+++
b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
@@ -3,8 +3,6 @@
package org.torproject.metrics.collector.bridgedescs;
-import static java.time.ZoneOffset.UTC;
-
import org.torproject.descriptor.BridgeExtraInfoDescriptor;
import org.torproject.descriptor.BridgeNetworkStatus;
import org.torproject.descriptor.BridgeServerDescriptor;
@@ -35,18 +33,12 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
-import java.security.GeneralSecurityException;
-import java.security.SecureRandom;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.Instant;
-import java.time.LocalDateTime;
-import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
@@ -89,26 +81,14 @@ public class SanitizedBridgesWriter extends CollecTorMain {
private Path inputDirectory;
- private boolean replaceIpAddressesWithHashes;
-
- private boolean persistenceProblemWithSecrets;
-
- private SortedMap secretsForHashingIpAddresses;
-
- private String bridgeSanitizingCutOffTimestamp;
-
- private boolean haveWarnedAboutInterval;
-
- private Path bridgeIpSecretsFile;
-
- private SecureRandom secureRandom;
-
private Path outputDirectory;
private Path recentDirectory;
private Path statsDirectory;
+ private SensitivePartsSanitizer sensitivePartsSanitizer;
+
@Override
public String module() {
return "bridgedescs";
@@ -128,90 +108,30 @@ public class SanitizedBridgesWriter extends CollecTorMain
{
.resolve(BRIDGE_DESCRIPTORS);
this.inputDirectory = config.getPath(Key.BridgeLocalOrigins);
this.statsDirectory = config.getPath(Key.StatsPath);
-this.replaceIpAddressesWithHashes =
+boolean replaceIpAddressesWithHashes =
config.getBool(Key.ReplaceIpAddressesWithHashes);
SimpleDateFormat rsyncCatFormat = new SimpleDateFormat(
"-MM-dd-HH-mm-ss");
this.rsyncCatString = rsyncCatFormat.format(
System.currentTimeMillis());
-/* Initialize secure random number generator if we need it. */
-if (this.replaceIpAddressesWithHashes) {
- try {
-this.secureRandom = SecureRandom.getInstance("SHA1PRNG", "SUN");
- } catch (GeneralSecurityException e) {
-logger.warn("Could not initialize secure "
-+ "random number generator! Not calculating any IP address "
-+ "hashes in this execution!", e);
-this.persistenceProblemWithSecrets = true;
- }
-}
-
-/* Read hex-encoded secrets for replacing IP addresses with hashes
- * from disk. */
-this.secretsForHashingIpAddresses = new TreeMap<>();
-this.bridgeIpSecretsFile = statsDirectory.resolve("bridge-ip-secrets");
-if (Files.exists(this.bridgeIpSecretsFile)) {
- try {
-for (String line : Files.readAllLines(this.bridgeIpSecretsFile)) {
- String[] parts = line.split(",");
- if ((line.length() != ("-MM,".length() + 31 * 2)
- && line.length() != ("-MM,".length() + 50 * 2)
- && line.length() != ("-MM,".length() + 83 * 2))
- || parts.length != 2) {
-logger.warn("Invalid line in bridge-ip-secrets file "
-+ "starting with '{}'! "
-+ "Not calculating any IP address hashes in this "
-+ "execution!", line.substring(0, 7));
-this.persistenceProblemWithSecrets = true;
-break;
- }
- String month = parts[0];
- byte[] secret = Hex.decodeHex(parts[1].toCharArray());
- this.secretsForHashingIpAddresses.put(month, secret);
-}
-if (!this.persistenceProblemWithSecrets) {
- logger.debug("Read {} secrets for
[tor-commits] [collector/master] Simplify the bridgedescs module.
commit 106852425554f6001f114ee711648798c78609ec
Author: Karsten Loesing
Date: Sat Nov 28 22:14:53 2020 +0100
Simplify the bridgedescs module.
The separation between BridgeSnapshotReader, BridgeDescriptorParser,
and SanitizedBridgesWriter doesn't make much sense anymore:
- BridgeSnapshotReader only has a constructor of more than 200 lines
of code.
- BridgeDescriptorParser actually only determines the descriptor type
and
- SanitizedBridgesWriter performs parsing and obfuscation.
There are better ways to structure this code. The first step in that
direction is to remove clutter by moving the code to read bridge
snapshots to SanitizedBridgesWriter and deleting the other two
classes.
Part of #20542.
---
.../bridgedescs/BridgeDescriptorParser.java| 55 -
.../bridgedescs/BridgeSnapshotReader.java | 248 -
.../bridgedescs/SanitizedBridgesWriter.java| 228 ++-
.../bridgedescs/BridgeDescriptorParserTest.java| 43
4 files changed, 223 insertions(+), 351 deletions(-)
diff --git
a/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeDescriptorParser.java
b/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeDescriptorParser.java
deleted file mode 100644
index b5e30bc..000
---
a/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeDescriptorParser.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/* Copyright 2010--2020 The Tor Project
- * See LICENSE for licensing information */
-
-package org.torproject.metrics.collector.bridgedescs;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.StringReader;
-import java.nio.charset.StandardCharsets;
-
-public class BridgeDescriptorParser {
-
- private SanitizedBridgesWriter sbw;
-
- private static final Logger logger = LoggerFactory.getLogger(
- BridgeDescriptorParser.class);
-
- /** Initializes a new bridge descriptor parser and links it to a
- * sanitized bridges writer to sanitize and store bridge descriptors. */
- public BridgeDescriptorParser(SanitizedBridgesWriter sbw) {
-if (null == sbw) {
- throw new IllegalArgumentException("SanitizedBridgesWriter has to be "
- + "provided, but was null.");
-}
-this.sbw = sbw;
- }
-
- /** Parses the first line of the given descriptor data to determine the
- * descriptor type and passes it to the sanitized bridges writer. */
- public void parse(byte[] allData, String dateTime,
- String authorityFingerprint) {
-try {
- BufferedReader br = new BufferedReader(new StringReader(
- new String(allData, StandardCharsets.US_ASCII)));
- String line = br.readLine();
- if (line == null) {
-return;
- }
- if (line.startsWith("router ")) {
-this.sbw.sanitizeAndStoreServerDescriptor(allData);
- } else if (line.startsWith("extra-info ")) {
-this.sbw.sanitizeAndStoreExtraInfoDescriptor(allData);
- } else {
-this.sbw.sanitizeAndStoreNetworkStatus(allData, dateTime,
-authorityFingerprint);
- }
-} catch (IOException e) {
- logger.warn("Could not parse or write bridge descriptor.", e);
-}
- }
-}
-
diff --git
a/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeSnapshotReader.java
b/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeSnapshotReader.java
deleted file mode 100644
index de9cd4b..000
---
a/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeSnapshotReader.java
+++ /dev/null
@@ -1,248 +0,0 @@
-/* Copyright 2010--2020 The Tor Project
- * See LICENSE for licensing information */
-
-package org.torproject.metrics.collector.bridgedescs;
-
-import org.apache.commons.codec.binary.Hex;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
-import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.StringReader;
-import java.nio.charset.StandardCharsets;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.Stack;
-import java.util.TreeSet;
-
-public class BridgeSnapshotReader {
-
- private static final Logger logger = LoggerFactory.getLogger(
- BridgeSnapshotReader.class);
-
- /**
- * Reads the half-hourly snapshots of bridge descriptors from Bifroest.
- */
- public BridgeSnap
[tor-commits] [collector/master] Make some minor optimizations to bridgedescs code.
commit 47a4c7a962de55ee8354c1c8605216965f68d116
Author: Karsten Loesing
Date: Mon Nov 30 22:45:22 2020 +0100
Make some minor optimizations to bridgedescs code.
Part of #20542.
---
.../collector/bridgedescs/DescriptorBuilder.java | 3 ++
.../bridgedescs/SanitizedBridgesWriter.java| 62 ++
2 files changed, 32 insertions(+), 33 deletions(-)
diff --git
a/src/main/java/org/torproject/metrics/collector/bridgedescs/DescriptorBuilder.java
b/src/main/java/org/torproject/metrics/collector/bridgedescs/DescriptorBuilder.java
index b4b63e7..946fcdb 100644
---
a/src/main/java/org/torproject/metrics/collector/bridgedescs/DescriptorBuilder.java
+++
b/src/main/java/org/torproject/metrics/collector/bridgedescs/DescriptorBuilder.java
@@ -96,4 +96,7 @@ class DescriptorBuilder {
return value;
}
+ public byte[] toBytes() {
+return this.toString().getBytes();
+ }
}
diff --git
a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
index 843aa40..77ab406 100644
---
a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
+++
b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
@@ -79,14 +79,10 @@ public class SanitizedBridgesWriter extends CollecTorMain {
private String rsyncCatString;
- private Path inputDirectory;
-
private Path outputDirectory;
private Path recentDirectory;
- private Path statsDirectory;
-
private SensitivePartsSanitizer sensitivePartsSanitizer;
@Override
@@ -106,8 +102,8 @@ public class SanitizedBridgesWriter extends CollecTorMain {
.resolve(BRIDGE_DESCRIPTORS);
this.recentDirectory = config.getPath(Key.RecentPath)
.resolve(BRIDGE_DESCRIPTORS);
-this.inputDirectory = config.getPath(Key.BridgeLocalOrigins);
-this.statsDirectory = config.getPath(Key.StatsPath);
+Path inputDirectory = config.getPath(Key.BridgeLocalOrigins);
+Path statsDirectory = config.getPath(Key.StatsPath);
boolean replaceIpAddressesWithHashes =
config.getBool(Key.ReplaceIpAddressesWithHashes);
SimpleDateFormat rsyncCatFormat = new SimpleDateFormat(
@@ -126,7 +122,7 @@ public class SanitizedBridgesWriter extends CollecTorMain {
}
// Import bridge descriptors
-this.readBridgeSnapshots(this.inputDirectory, this.statsDirectory);
+this.readBridgeSnapshots(inputDirectory, statsDirectory);
// Finish writing sanitized bridge descriptors to disk
if (replaceIpAddressesWithHashes) {
@@ -362,11 +358,16 @@ public class SanitizedBridgesWriter extends CollecTorMain
{
}
/* Parse the given network status line by line. */
-DescriptorBuilder header = new DescriptorBuilder();
boolean includesFingerprintLine = false;
-SortedMap scrubbedLines = new TreeMap<>();
+DescriptorBuilder scrubbed = new DescriptorBuilder();
+scrubbed.append(Annotation.Status.toString());
+SortedMap scrubbedEntries = new TreeMap<>();
+StringBuilder publishedStringBuilder = new StringBuilder();
+scrubbed.append("published ").append(publishedStringBuilder).newLine();
+DescriptorBuilder header = new DescriptorBuilder();
+scrubbed.append(header);
+
try {
- DescriptorBuilder scrubbed = new DescriptorBuilder();
BufferedReader br = new BufferedReader(new StringReader(new String(
data, StandardCharsets.US_ASCII)));
String line;
@@ -374,6 +375,7 @@ public class SanitizedBridgesWriter extends CollecTorMain {
byte[] fingerprintBytes = null;
String descPublicationTime = null;
String hashedBridgeIdentityHex = null;
+ DescriptorBuilder scrubbedEntry = new DescriptorBuilder();
while ((line = br.readLine()) != null) {
/* Use publication time from "published" line instead of the
@@ -403,10 +405,10 @@ public class SanitizedBridgesWriter extends CollecTorMain
{
} else if (line.startsWith("r ")) {
/* Clear buffer from previously scrubbed lines. */
- if (scrubbed.hasContent()) {
-String scrubbedLine = scrubbed.toString();
-scrubbedLines.put(hashedBridgeIdentityHex, scrubbedLine);
-scrubbed = new DescriptorBuilder();
+ if (scrubbedEntry.hasContent()) {
+scrubbedEntries.put(hashedBridgeIdentityHex,
+scrubbedEntry.toString());
+scrubbedEntry = new DescriptorBuilder();
}
/* Parse the relevant parts of this r line. */
@@ -452,7 +454,7 @@ public class SanitizedBridgesWriter extends CollecTorMain {
orPort, fingerprintBytes, descPublicationTime);
String scrubbedDirPort = this.sensitivePartsSanitizer.scrubTcpPort(
dirPort, fingerprintBytes, descPublicationTime);
- sc
[tor-commits] [collector/master] Update most of the bridgedescs module to NIO.
commit c0ee1a6cf76f6f6b1677edccc1dc7e4055de50e9
Author: Karsten Loesing
Date: Sun Nov 29 00:05:47 2020 +0100
Update most of the bridgedescs module to NIO.
Replace all File references with their Path equivalents, and use Files
methods wherever feasible.
Part of #20542.
---
.../bridgedescs/SanitizedBridgesWriter.java| 288 +
1 file changed, 127 insertions(+), 161 deletions(-)
diff --git
a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
index 8db7db5..34156c2 100644
---
a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
+++
b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
@@ -26,16 +26,15 @@ import org.slf4j.LoggerFactory;
import java.io.BufferedInputStream;
import java.io.BufferedReader;
-import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileReader;
-import java.io.FileWriter;
import java.io.IOException;
+import java.io.InputStream;
import java.io.StringReader;
import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
import java.security.GeneralSecurityException;
import java.security.SecureRandom;
import java.text.ParseException;
@@ -45,7 +44,6 @@ import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -89,12 +87,7 @@ public class SanitizedBridgesWriter extends CollecTorMain {
private String rsyncCatString;
- private File bridgeDirectoriesDirectory;
-
- /**
- * Output directory for writing sanitized bridge descriptors.
- */
- private File sanitizedBridgesDirectory;
+ private Path inputDirectory;
private boolean replaceIpAddressesWithHashes;
@@ -106,13 +99,15 @@ public class SanitizedBridgesWriter extends CollecTorMain {
private boolean haveWarnedAboutInterval;
- private File bridgeIpSecretsFile;
+ private Path bridgeIpSecretsFile;
private SecureRandom secureRandom;
- private String outputPathName;
+ private Path outputDirectory;
+
+ private Path recentDirectory;
- private String recentPathName;
+ private Path statsDirectory;
@Override
public String module() {
@@ -127,25 +122,12 @@ public class SanitizedBridgesWriter extends CollecTorMain
{
@Override
protected void startProcessing() throws ConfigurationException {
-outputPathName = Paths.get(config.getPath(Key.OutputPath).toString(),
-BRIDGE_DESCRIPTORS).toString();
-recentPathName = Paths.get(config.getPath(Key.RecentPath).toString(),
-BRIDGE_DESCRIPTORS).toString();
-File bridgeDirectoriesDirectory =
-config.getPath(Key.BridgeLocalOrigins).toFile();
-File sanitizedBridgesDirectory = new File(outputPathName);
-File statsDirectory = config.getPath(Key.StatsPath).toFile();
-
-if (bridgeDirectoriesDirectory == null
-|| sanitizedBridgesDirectory == null || statsDirectory == null) {
- throw new ConfigurationException("BridgeSnapshotsDirectory, "
- + "SanitizedBridgesWriteDirectory, StatsPath should be set. "
- + "Please, edit the 'collector.properties' file.");
-}
-
-/* Memorize argument values. */
-this.bridgeDirectoriesDirectory = bridgeDirectoriesDirectory;
-this.sanitizedBridgesDirectory = sanitizedBridgesDirectory;
+this.outputDirectory = config.getPath(Key.OutputPath)
+.resolve(BRIDGE_DESCRIPTORS);
+this.recentDirectory = config.getPath(Key.RecentPath)
+.resolve(BRIDGE_DESCRIPTORS);
+this.inputDirectory = config.getPath(Key.BridgeLocalOrigins);
+this.statsDirectory = config.getPath(Key.StatsPath);
this.replaceIpAddressesWithHashes =
config.getBool(Key.ReplaceIpAddressesWithHashes);
SimpleDateFormat rsyncCatFormat = new SimpleDateFormat(
@@ -168,13 +150,10 @@ public class SanitizedBridgesWriter extends CollecTorMain
{
/* Read hex-encoded secrets for replacing IP addresses with hashes
* from disk. */
this.secretsForHashingIpAddresses = new TreeMap<>();
-this.bridgeIpSecretsFile = new File(statsDirectory,
-"bridge-ip-secrets");
-if (this.bridgeIpSecretsFile.exists()) {
- try (BufferedReader br = new BufferedReader(new FileReader(
- this.bridgeIpSecretsFile))) {
-String line;
-while ((line = br.readLine()) != null) {
+this.bridgeIpSecretsFile = statsDirectory.resolve("bridge-ip-secrets");
+if (Files.exists(this.bridgeIpSecretsFile)) {
+
[tor-commits] [collector/master] Add change log entry for #34030 fix.
commit f1c7198ac4a414d5f25974997662a4b2645d8f95 Author: Karsten Loesing Date: Sat Nov 28 20:35:22 2020 +0100 Add change log entry for #34030 fix. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e292f9a..2928937 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ * Medium changes - Clean up descriptors written to the `out/` directory by deleting files that are older than seven weeks. + - Correctly index files that are moved away and back. # Changes in version 1.16.1 - 2020-08-16 ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/master] Fix minor issue with cleaning up directories.
commit cd15f3446376847f359040016fb95791dabfce15
Author: Karsten Loesing
Date: Sat Nov 28 11:10:30 2020 +0100
Fix minor issue with cleaning up directories.
One of the previously made changes to cleaning up directories was that
empty directories were deleted. This was necessary, because otherwise
there would be a growing number of directories as files get deleted
after reaching an age of seven weeks.
However, this change should not have included deleting the cleaned up
directory itself. In practice, this will not happen. But in tests it's
certainly possible that a directory is empty and then gets deleted.
This leads to all sorts of problems in tests.
The fix is to limit deleting empty directories to subdirectories.
That's what this commit does.
---
.../org/torproject/metrics/collector/persist/PersistenceUtils.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git
a/src/main/java/org/torproject/metrics/collector/persist/PersistenceUtils.java
b/src/main/java/org/torproject/metrics/collector/persist/PersistenceUtils.java
index e787c39..2b7621d 100644
---
a/src/main/java/org/torproject/metrics/collector/persist/PersistenceUtils.java
+++
b/src/main/java/org/torproject/metrics/collector/persist/PersistenceUtils.java
@@ -132,7 +132,8 @@ public class PersistenceUtils {
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc)
throws IOException {
-if (!Files.list(dir).findFirst().isPresent()) {
+if (!pathToClean.equals(dir)
+&& !Files.list(dir).findFirst().isPresent()) {
Files.delete(dir);
}
return FileVisitResult.CONTINUE;
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/master] Correctly index files that are moved away and back.
commit 42a0dd280924cdf1ad5c2be3c6c18536a197c88f
Author: Karsten Loesing
Date: Sat Nov 28 11:14:21 2020 +0100
Correctly index files that are moved away and back.
The indexer did not handle a (mostly theoretic) edge case of a file
being moved away and then moved back shortly after. In such a case the
file should not be marked for deletion anymore and it should be
included in the index again. That's what this commit does.
The other minor changes to unit tests are just cosmetic.
Fixes #34030.
---
.../metrics/collector/indexer/CreateIndexJson.java | 6 ++
.../collector/indexer/CreateIndexJsonTest.java | 23 +++---
2 files changed, 26 insertions(+), 3 deletions(-)
diff --git
a/src/main/java/org/torproject/metrics/collector/indexer/CreateIndexJson.java
b/src/main/java/org/torproject/metrics/collector/indexer/CreateIndexJson.java
index 6613e9f..3d86947 100644
---
a/src/main/java/org/torproject/metrics/collector/indexer/CreateIndexJson.java
+++
b/src/main/java/org/torproject/metrics/collector/indexer/CreateIndexJson.java
@@ -426,6 +426,12 @@ public class CreateIndexJson extends CollecTorMain {
/* We do have index results, but we don't have a link yet, so we're
* going to create a link. */
linksToCreate.put(linkPath, filePath);
+ if (null != fileNode.markedForDeletion) {
+/* We had already marked the link for deletion, but given that the
+ * original file has returned, we're going to list this file again
+ * and not delete the link in the future. */
+fileNode.markedForDeletion = null;
+ }
} else {
String linkLastModified = dateTimeFormatter
.format(Files.getLastModifiedTime(linkPath).toInstant());
diff --git
a/src/test/java/org/torproject/metrics/collector/indexer/CreateIndexJsonTest.java
b/src/test/java/org/torproject/metrics/collector/indexer/CreateIndexJsonTest.java
index 9c01293..a176d27 100644
---
a/src/test/java/org/torproject/metrics/collector/indexer/CreateIndexJsonTest.java
+++
b/src/test/java/org/torproject/metrics/collector/indexer/CreateIndexJsonTest.java
@@ -404,8 +404,7 @@ public class CreateIndexJsonTest {
createFile(recentExitListFilePath, Instant.parse("2016-09-20T13:02:00Z"));
writeIndexJson(recentExitListIndexJsonString);
startProcessing(firstExecution);
-startProcessing(secondExecution);
-assertEquals(recentExitListIndexJsonString, readIndexJson());
+assertTrue(this.indexerTasks.isEmpty());
}
/**
@@ -422,7 +421,6 @@ public class CreateIndexJsonTest {
deleteFile(recentExitListFilePath);
startProcessing(secondExecution);
assertEquals(emptyIndexJsonString, readIndexJson());
-fileExists(recentExitListLinkPath);
assertTrue(fileExists(recentExitListLinkPath));
startProcessing(thirdExecution);
assertFalse(fileExists(recentExitListLinkPath));
@@ -518,5 +516,24 @@ public class CreateIndexJsonTest {
startProcessing(thirdExecution);
assertTrue(this.indexerTasks.isEmpty());
}
+
+ /**
+ * Test whether a file that was previously contained in the index and deleted
+ * or moved away and that is later recreated or moved back to its original
+ * location is included in the index again and still has a corresponding link
+ * three hours later.
+ */
+ @Test
+ public void testMoveBackFile() {
+writeIndexJson(recentExitListIndexJsonString);
+startProcessing(firstExecution);
+createFile(recentExitListFilePath, Instant.parse("2016-09-20T13:02:00Z"));
+startProcessing(secondExecution);
+assertTrue(this.indexerTasks.isEmpty());
+assertTrue(Files.exists(recentExitListLinkPath));
+assertEquals(recentExitListIndexJsonString, readIndexJson());
+startProcessing(thirdExecution);
+assertTrue(Files.exists(recentExitListLinkPath));
+ }
}
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/master] Bump version to 1.16.1-dev.
commit 91d10a09608df31f3ef50f9ec3faa156d8bc2713 Author: Karsten Loesing Date: Wed Nov 25 16:08:39 2020 +0100 Bump version to 1.16.1-dev. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index e6e727a..5dcb29d 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,7 @@ - + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/master] Delete files in out/ that are older than 7 weeks.
commit 66ddc4d7d996ad2877aea44ea03982f14f069545
Author: Karsten Loesing
Date: Wed Nov 25 16:09:14 2020 +0100
Delete files in out/ that are older than 7 weeks.
Fixes #21219.
---
CHANGELOG.md | 7 +++
.../bridgedb/BridgedbMetricsProcessor.java | 38 --
.../bridgedescs/SanitizedBridgesWriter.java| 37 +
.../BridgePoolAssignmentsProcessor.java| 33
.../collector/exitlists/ExitListDownloader.java| 27 --
.../collector/onionperf/OnionPerfDownloader.java | 37 ++---
.../collector/persist/PersistenceUtils.java| 50 +++---
.../collector/relaydescs/ArchiveWriter.java| 61 +-
.../snowflake/SnowflakeStatsDownloader.java| 33 +---
.../metrics/collector/sync/SyncPersistence.java| 7 +--
.../collector/webstats/SanitizeWeblogs.java| 12 +++--
.../collector/persist/PersistUtilsTest.java| 32
12 files changed, 182 insertions(+), 192 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ff2e9e7..e292f9a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,10 @@
+# Changes in version 1.??.? - 2020-1?-??
+
+ * Medium changes
+ - Clean up descriptors written to the `out/` directory by deleting
+files that are older than seven weeks.
+
+
# Changes in version 1.16.1 - 2020-08-16
* Medium changes
diff --git
a/src/main/java/org/torproject/metrics/collector/bridgedb/BridgedbMetricsProcessor.java
b/src/main/java/org/torproject/metrics/collector/bridgedb/BridgedbMetricsProcessor.java
index 0073ee3..d05aa9c 100644
---
a/src/main/java/org/torproject/metrics/collector/bridgedb/BridgedbMetricsProcessor.java
+++
b/src/main/java/org/torproject/metrics/collector/bridgedb/BridgedbMetricsProcessor.java
@@ -12,6 +12,7 @@ import
org.torproject.metrics.collector.conf.ConfigurationException;
import org.torproject.metrics.collector.conf.Key;
import org.torproject.metrics.collector.cron.CollecTorMain;
import org.torproject.metrics.collector.persist.BridgedbMetricsPersistence;
+import org.torproject.metrics.collector.persist.PersistenceUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -23,9 +24,7 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
-import java.util.Arrays;
import java.util.SortedSet;
-import java.util.Stack;
import java.util.TreeSet;
public class BridgedbMetricsProcessor extends CollecTorMain {
@@ -127,10 +126,10 @@ public class BridgedbMetricsProcessor extends
CollecTorMain {
descriptor.getClass(), descriptor.getDescriptorFile());
}
}
-logger.info("Cleaning up directory {} containing recent files.",
-this.recentPathName);
+logger.info("Cleaning up directories {} and {}.",
+this.recentPathName, this.outputPathName);
this.writeProcessedFiles(this.parsedBridgedbMetricsFile, processedFiles);
-this.cleanUpRsyncDirectory();
+this.cleanUpDirectories();
logger.info("Finished processing BridgeDB statistics file(s).");
}
@@ -175,28 +174,13 @@ public class BridgedbMetricsProcessor extends
CollecTorMain {
}
/**
- * Delete all files from the rsync directory that have not been modified in
- * the last three days.
+ * Delete all files from the rsync (out) directory that have not been
modified
+ * in the last three days (seven weeks).
*/
- public void cleanUpRsyncDirectory() {
-Instant cutOff = Instant.now().minus(3L, ChronoUnit.DAYS);
-Stack allFiles = new Stack<>();
-allFiles.add(new File(this.recentPathName));
-while (!allFiles.isEmpty()) {
- File file = allFiles.pop();
- if (file.isDirectory()) {
-File[] filesInDirectory = file.listFiles();
-if (null != filesInDirectory) {
- allFiles.addAll(Arrays.asList(filesInDirectory));
-}
- } else if (Instant.ofEpochMilli(file.lastModified()).isBefore(cutOff)) {
-try {
- Files.deleteIfExists(file.toPath());
-} catch (IOException e) {
- logger.warn("Unable to delete file {} that is apparently older than "
- + "three days.", file, e);
-}
- }
-}
+ private void cleanUpDirectories() {
+PersistenceUtils.cleanDirectory(Paths.get(this.recentPathName),
+Instant.now().minus(3, ChronoUnit.DAYS).toEpochMilli());
+PersistenceUtils.cleanDirectory(Paths.get(this.outputPathName),
+Instant.now().minus(49, ChronoUnit.DAYS).toEpochMilli());
}
}
diff --git
a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
index b8e7f2d..62288ad 100644
---
a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.ja
[tor-commits] [metrics-web/master] Update from Metrics Timeline, and add links.
commit e54b73882132911fa9ec0809bba0823fb562d6a4
Author: Karsten Loesing
Date: Tue Sep 15 09:38:32 2020 +0200
Update from Metrics Timeline, and add links.
---
src/main/resources/web/json/news.json | 19 ---
src/main/resources/web/jsps/graph.jsp | 4 ++--
src/main/resources/web/jsps/news.jsp | 2 +-
3 files changed, 19 insertions(+), 6 deletions(-)
diff --git a/src/main/resources/web/json/news.json
b/src/main/resources/web/json/news.json
index 43c28e8..901b00c 100644
--- a/src/main/resources/web/json/news.json
+++ b/src/main/resources/web/json/news.json
@@ -5,7 +5,7 @@
"description" : "Shutdown of default obfs4 bridge frosty.",
"links" : [ {
"label" : "ticket",
-"target" :
"https://gitlab.torproject.org/tpo/applications/tor-browser-build/-/issues/40066;
+"target" :
"https://bugs.torproject.org/tpo/applications/tor-browser-build/40066;
} ]
}, {
"start" : "2020-08-13",
@@ -14,7 +14,7 @@
"description" : "Deployed version 0.4.1 of the Snowflake WebExtension, with
NAT type detection in the browser.",
"links" : [ {
"label" : "comment",
-"target" :
"https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake-webext/-/issues/13#note_2705805;
+"target" :
"https://bugs.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake-webext/13#note_2705805;
} ]
}, {
"start" : "2020-07-06",
@@ -23,7 +23,7 @@
"description" : "Deployed NAT-aware matching at the Snowflake broker.",
"links" : [ {
"label" : "comment",
-"target" :
"https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/-/issues/34129#note_2685268;
+"target" :
"https://bugs.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/34129#note_2685268;
} ]
}, {
"start" : "2020-06-30",
@@ -3540,6 +3540,19 @@
"label" : "commit",
"target" :
"https://gitweb.torproject.org/tor.git/commit/?id=1496056c121d02193433d98173213e52eb3e90a7;
} ]
+}, {
+ "start" : "2015-12-31",
+ "end" : "2020-06-04",
+ "places" : [ "mx" ],
+ "short_description" : "The Telmex ISP in Mexico blocks seven directory
authorities.",
+ "description" : "The Telmex ISP in Mexico blocks seven directory
authorities.",
+ "links" : [ {
+"label" : "mailing list post",
+"target" :
"https://lists.torproject.org/pipermail/tor-relays/2016-January/008491.html;
+ }, {
+"label" : "article",
+"target" :
"https://globalvoices.org/2020/09/08/we-made-the-largest-mexican-telecommunications-operator-stop-blocking-secure-internet/;
+ } ]
}, {
"start" : "2015-12-25",
"protocols" : [ "meek" ],
diff --git a/src/main/resources/web/jsps/graph.jsp
b/src/main/resources/web/jsps/graph.jsp
index 94aa784..8d601e5 100644
--- a/src/main/resources/web/jsps/graph.jsp
+++ b/src/main/resources/web/jsps/graph.jsp
@@ -160,8 +160,8 @@
Related events
- The following events have been manually collected on
- https://trac.torproject.org/projects/tor/wiki/doc/MetricsTimeline;
target="_blank">this wiki page
+ The following events have been manually collected in the
+ https://gitlab.torproject.org/tpo/metrics/timeline;
target="_blank">metrics-timeline Git repository
and might be related to the displayed graph.
diff --git a/src/main/resources/web/jsps/news.jsp
b/src/main/resources/web/jsps/news.jsp
index abde1e0..d78cde9 100644
--- a/src/main/resources/web/jsps/news.jsp
+++ b/src/main/resources/web/jsps/news.jsp
@@ -19,7 +19,7 @@
News #
- You're a journalist or more generally a person who wants to know
what's going on in the Tor network? We're collecting unusual events in the Tor
network together with any insights we have into what we think has happened.
+ You're a journalist or more generally a person who wants to know
what's going on in the Tor network? We're collecting unusual events in the Tor
network together with any insights we have into what we think has happened in
the https://gitlab.torproject.org/tpo/metrics/timeline;
target="_blank">metrics-timeline Git repository.
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/master] Update OnionPerf link.
commit c91b83230639d6779b06872a600e361b3cb388d1 Author: Karsten Loesing Date: Mon Sep 21 15:58:05 2020 +0200 Update OnionPerf link. --- src/main/resources/web/jsps/sources.jsp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/resources/web/jsps/sources.jsp b/src/main/resources/web/jsps/sources.jsp index 1f2930c..eb0ebc6 100644 --- a/src/main/resources/web/jsps/sources.jsp +++ b/src/main/resources/web/jsps/sources.jsp @@ -34,8 +34,7 @@ https://gitweb.torproject.org/user/phw/exitmap.git; target="_blank">Exitmap is a fast and extensible scanner for Tor exit relays. https://www.torproject.org/projects/tordnsel.html.en; target="_blank">TorDNSEL publishes lists of IP addresses of multi-homed Tor exits. -https://gitweb.torproject.org/torperf.git; target="_blank">Torperf measures Tor performance with a set of utilities and Python scripts. -https://github.com/robgjansen/onionperf; target="_blank">OnionPerf measures the performance of onion services. +https://gitlab.torproject.org/tpo/metrics/onionperf; target="_blank">OnionPerf measures Tor performance with a set of utilities and Python scripts. https://ooni.torproject.org/; target="_blank">OONI detects censorship, surveillance, and traffic manipulation on the internet. https://gitweb.torproject.org/user/phw/sybilhunter.git/; target="_blank">Sybilhunter attempts to detect Sybil attacks on the Tor network. ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/master] Append to existing "filters" field.
commit c41c0b74c47442dc6f6d23f49a012729f71aee49
Author: Karsten Loesing
Date: Wed Sep 16 12:01:07 2020 +0200
Append to existing "filters" field.
Before this change we would have replaced an existing "filters" field
with the filters applied in this run. But if the user runs `onionperf
filter` more than once to apply different filters, we'll want to list
all filters in the "filters" field.
This change also removes a couple of `self.`s, in particular
`self.analysis` and `self.filters`, because one Filtering object can
apply filters to more than one analysis.
---
onionperf/filtering.py | 20 +---
1 file changed, 9 insertions(+), 11 deletions(-)
diff --git a/onionperf/filtering.py b/onionperf/filtering.py
index e8ea164..15d9c19 100644
--- a/onionperf/filtering.py
+++ b/onionperf/filtering.py
@@ -7,7 +7,6 @@
import re
from onionperf.analysis import OPAnalysis
-from collections import defaultdict
class Filtering(object):
@@ -15,7 +14,6 @@ class Filtering(object):
self.fingerprints_to_include = None
self.fingerprints_to_exclude = None
self.fingerprint_pattern = re.compile("\$?([0-9a-fA-F]{40})")
-self.filters = defaultdict(list)
def include_fingerprints(self, path):
self.fingerprints_to_include = []
@@ -40,11 +38,12 @@ class Filtering(object):
def filter_tor_circuits(self, analysis):
if self.fingerprints_to_include is None and
self.fingerprints_to_exclude is None:
return
-self.filters["tor/circuits"] = []
+filters = analysis.json_db.setdefault("filters", {})
+tor_circuits_filters = filters.setdefault("tor/circuits", [])
if self.fingerprints_to_include:
- self.filters["tor/circuits"].append({"name":
"include_fingerprints", "filepath": self.fingerprints_to_include_path })
+ tor_circuits_filters.append({"name": "include_fingerprints",
"filepath": self.fingerprints_to_include_path })
if self.fingerprints_to_exclude:
- self.filters["tor/circuits"].append({"name":
"exclude_fingerprints", "filepath": self.fingerprints_to_exclude_path })
+ tor_circuits_filters.append({"name": "exclude_fingerprints",
"filepath": self.fingerprints_to_exclude_path })
for source in analysis.get_nodes():
tor_circuits = analysis.get_tor_circuits(source)
filtered_circuit_ids = []
@@ -68,10 +67,9 @@ class Filtering(object):
tor_circuits[circuit_id] =
dict(sorted(tor_circuit.items()))
def apply_filters(self, input_path, output_dir, output_file):
-self.analysis = OPAnalysis.load(filename=input_path)
-self.filter_tor_circuits(self.analysis)
-self.analysis.json_db["filters"] = self.filters
-self.analysis.json_db["version"] = '4.0'
-self.analysis.json_db = dict(sorted(self.analysis.json_db.items()))
-self.analysis.save(filename=output_file, output_prefix=output_dir,
sort_keys=False)
+analysis = OPAnalysis.load(filename=input_path)
+self.filter_tor_circuits(analysis)
+analysis.json_db["version"] = '4.0'
+analysis.json_db = dict(sorted(analysis.json_db.items()))
+analysis.save(filename=output_file, output_prefix=output_dir,
sort_keys=False)
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/master] Only try to load analysis files in an input directory
commit d21d41e4d504b48ac5617530830da320a89a9eed Author: Ana Custura Date: Sat Aug 29 15:10:32 2020 +0100 Only try to load analysis files in an input directory --- onionperf/onionperf | 11 ++- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/onionperf/onionperf b/onionperf/onionperf index 7c16aea..e3f49c8 100755 --- a/onionperf/onionperf +++ b/onionperf/onionperf @@ -458,11 +458,12 @@ def filter(args): output_dir, output_file = os.path.split(output_path) filtering.apply_filters(input_path=input_path, output_dir=output_dir, output_file=output_file) else: -for dirpath, dirnames, filenames in os.walk(input_path): -for filename in filenames: -input_file = os.path.join(dirpath, filename) -output_dir = os.path.join(output_path, os.path.relpath(dirpath, input_path)) -filtering.apply_filters(input_path=input_file, output_dir=output_dir, output_file=filename) +from onionperf import reprocessing +analyses = reprocessing.collect_logs(input_path, '*onionperf.analysis.*') +for analysis in analyses: +full_output_path = os.path.join(output_path, os.path.relpath(analysis, input_path)) +output_dir, output_file = os.path.split(full_output_path) +filtering.apply_filters(input_path=analysis, output_dir=output_dir, output_file=output_file) def visualize(args): from onionperf.visualization import TGenVisualization ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/master] Change filter mode to filter Tor circuits.
commit 9d0c80561b84905e72b41758dfcb7b712f5f407f
Author: Karsten Loesing
Date: Mon Aug 31 11:30:53 2020 +0200
Change filter mode to filter Tor circuits.
This new filter mode removes Tor circuits that don't match the
provided fingerprints and leaves TGen transfers/streams untouched. At
the same time the visualize mode only includes TGen transfers/streams
with an existing mapping between TGen transfers/streams and Tor
streams/circuits.
This patch changes the default behavior of the visualize mode. The
original behavior of visualizing TGen transfers/streams *without* an
existing mapping to Tor streams/circuits can be selected with the
--outer-join switch, even though that's rather an edge use case.
Another minor change is that the filtered analysis files is not
written with sort_keys=True anymore, which would have produced a newly
sorted file with keys in alphabetic order rather than the original
insert order. The result is an actually useful diff.
---
onionperf/analysis.py | 13 +---
onionperf/filtering.py | 49 --
onionperf/onionperf| 24 +--
onionperf/visualization.py | 33 ---
4 files changed, 53 insertions(+), 66 deletions(-)
diff --git a/onionperf/analysis.py b/onionperf/analysis.py
index b2f483f..49c109c 100644
--- a/onionperf/analysis.py
+++ b/onionperf/analysis.py
@@ -62,13 +62,7 @@ class OPAnalysis(Analysis):
self.json_db['data'][self.nickname]["tgen"].pop("stream_summary")
self.did_analysis = True
-def set_tgen_transfers(self, node, tgen_transfers):
-self.json_db['data'][node]['tgen']['transfers'] = tgen_transfers
-
-def set_tgen_streams(self, node, tgen_streams):
-self.json_db['data'][node]['tgen']['streams'] = tgen_streams
-
-def save(self, filename=None, output_prefix=os.getcwd(), do_compress=True,
date_prefix=None):
+def save(self, filename=None, output_prefix=os.getcwd(), do_compress=True,
date_prefix=None, sort_keys=True):
if filename is None:
base_filename = "onionperf.analysis.json.xz"
if date_prefix is not None:
@@ -85,7 +79,7 @@ class OPAnalysis(Analysis):
logging.info("saving analysis results to {0}".format(filepath))
outf = util.FileWritable(filepath, do_compress=do_compress)
-json.dump(self.json_db, outf, sort_keys=True, separators=(',', ': '),
indent=2)
+json.dump(self.json_db, outf, sort_keys=sort_keys, separators=(',', ':
'), indent=2)
outf.close()
logging.info("done!")
@@ -109,6 +103,9 @@ class OPAnalysis(Analysis):
except:
return None
+def set_tor_circuits(self, node, tor_circuits):
+self.json_db['data'][node]['tor']['circuits'] = tor_circuits
+
def get_tor_streams(self, node):
try:
return self.json_db['data'][node]['tor']['streams']
diff --git a/onionperf/filtering.py b/onionperf/filtering.py
index 9e7b34f..1b614d6 100644
--- a/onionperf/filtering.py
+++ b/onionperf/filtering.py
@@ -38,41 +38,11 @@ class Filtering(object):
if self.fingerprints_to_include is None and
self.fingerprints_to_exclude is None:
return
for source in self.analysis.get_nodes():
-tor_streams_by_source_port = {}
-tor_streams = self.analysis.get_tor_streams(source)
-for tor_stream in tor_streams.values():
-if "source" in tor_stream and ":" in tor_stream["source"]:
-source_port = tor_stream["source"].split(":")[1]
-tor_streams_by_source_port.setdefault(source_port,
[]).append(tor_stream)
tor_circuits = self.analysis.get_tor_circuits(source)
-tgen_streams = self.analysis.get_tgen_streams(source)
-tgen_transfers = self.analysis.get_tgen_transfers(source)
-retained_tgen_streams = {}
-retained_tgen_transfers = {}
-while tgen_streams or tgen_transfers:
-stream_id = None
-transfer_id = None
-source_port = None
-unix_ts_end = None
+filtered_circuit_ids = []
+for circuit_id, tor_circuit in tor_circuits.items():
keep = False
-if tgen_streams:
-stream_id, stream_data = tgen_streams.popitem()
-if "local" in stream_data["transport_info"] and
len(stream_data["transport_info"]["local"].split(":")) > 2:
-source_port =
stream_data["transport_info"]["local"].split(":")[2]
-if "unix_ts_end" in stream_data:
-
[tor-commits] [onionperf/master] Move filters and filter metadata to analysis files
commit 95b749a8fc690825c0a828b8473c58faea7ad912
Author: Ana Custura
Date: Thu Sep 10 01:51:54 2020 +0100
Move filters and filter metadata to analysis files
---
onionperf/filtering.py | 25 ++---
onionperf/onionperf| 9 +
onionperf/visualization.py | 14 +-
3 files changed, 28 insertions(+), 20 deletions(-)
diff --git a/onionperf/filtering.py b/onionperf/filtering.py
index 1b614d6..c008c03 100644
--- a/onionperf/filtering.py
+++ b/onionperf/filtering.py
@@ -7,6 +7,7 @@
import re
from onionperf.analysis import OPAnalysis
+from collections import defaultdict
class Filtering(object):
@@ -14,9 +15,11 @@ class Filtering(object):
self.fingerprints_to_include = None
self.fingerprints_to_exclude = None
self.fingerprint_pattern = re.compile("\$?([0-9a-fA-F]{40})")
+self.filters = defaultdict(list)
def include_fingerprints(self, path):
self.fingerprints_to_include = []
+self.fingerprints_to_include_path = path
with open(path, 'rt') as f:
for line in f:
fingerprint_match = self.fingerprint_pattern.match(line)
@@ -26,6 +29,7 @@ class Filtering(object):
def exclude_fingerprints(self, path):
self.fingerprints_to_exclude = []
+self.fingerprints_to_exclude_path = path
with open(path, 'rt') as f:
for line in f:
fingerprint_match = self.fingerprint_pattern.match(line)
@@ -33,12 +37,16 @@ class Filtering(object):
fingerprint = fingerprint_match.group(1).upper()
self.fingerprints_to_exclude.append(fingerprint)
-def apply_filters(self, input_path, output_dir, output_file):
-self.analysis = OPAnalysis.load(filename=input_path)
+def filter_tor_circuits(self, analysis):
if self.fingerprints_to_include is None and
self.fingerprints_to_exclude is None:
return
-for source in self.analysis.get_nodes():
-tor_circuits = self.analysis.get_tor_circuits(source)
+self.filters["tor/circuits"] = []
+if self.fingerprints_to_include:
+ self.filters["tor/circuits"].append({"name":
"include_fingerprints", "filepath": self.fingerprints_to_include_path })
+if self.fingerprints_to_exclude:
+ self.filters["tor/circuits"].append({"name":
"exclude_fingerprints", "filepath": self.fingerprints_to_exclude_path })
+for source in analysis.get_nodes():
+tor_circuits = analysis.get_tor_circuits(source)
filtered_circuit_ids = []
for circuit_id, tor_circuit in tor_circuits.items():
keep = False
@@ -56,8 +64,11 @@ class Filtering(object):
keep = False
break
if not keep:
-filtered_circuit_ids.append(circuit_id)
-for circuit_id in filtered_circuit_ids:
-del(tor_circuits[circuit_id])
+tor_circuits[circuit_id]["filtered"] = True
+
+def apply_filters(self, input_path, output_dir, output_file):
+self.analysis = OPAnalysis.load(filename=input_path)
+self.filter_tor_circuits(self.analysis)
+self.analysis.json_db["filters"] = self.filters
self.analysis.save(filename=output_file, output_prefix=output_dir,
sort_keys=False)
diff --git a/onionperf/onionperf b/onionperf/onionperf
index 1efa8cb..108af4e 100755
--- a/onionperf/onionperf
+++ b/onionperf/onionperf
@@ -342,13 +342,6 @@ files generated by this script will be written""",
required="True",
action=PathStringArgsAction, dest="datasets")
-visualize_parser.add_argument('--outer-join',
-help="""Include measurements without an existing mapping between TGen
-transfers/streams and Tor streams/circuits, which is the
-equivalent of an outer join in the database sense""",
-action="store_true", dest="outer_join",
-default=False)
-
visualize_parser.add_argument('-p', '--prefix',
help="a STRING filename prefix for graphs we generate",
metavar="STRING", type=str,
@@ -489,7 +482,7 @@ def visualize(args):
if analysis is not None:
analyses.append(analysis)
tgen_viz.add_dataset(analyses, label)
-tgen_viz.plot_all(args.prefix, outer_join=args.outer_join)
+tgen_viz.plot_all(args.prefix)
def type_nonnegative_integer(value):
i = int(value)
diff --git a/onionperf/visualization.py b/onionperf/visualization.py
index 0f69879..f5bc03f 100644
--- a/onionperf/visualization.py
+++ b/onionperf/visualization.py
@@ -31,11 +31,11 @@ class Visualization(object, metaclass=ABCMeta):
class TGenVisualization(Visualization):
-def plot_all(self, output_prefix, outer_join=False):
+def plot_all(self, output_prefix):
if len(self.datasets) > 0:
[tor-commits] [onionperf/master] Update README and CHANGELOG
commit 1b638e34cb0476148dcf93b43b58b54a30c27428 Author: Ana Custura Date: Thu Sep 10 14:46:10 2020 +0100 Update README and CHANGELOG --- CHANGELOG.md | 5 + README.md| 10 +++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebd43e1..a7d5005 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,11 @@ - Add a new `onionperf filter` mode that takes an OnionPerf analysis results file or directory as input, applies filters, and produces new OnionPerf analysis results file(s) as output. + - Bump the analysis version number to 4.0 for new analyses and analyses + produced by the `onionperf filter` mode + - Analyses produced by the `onionperf filter` mode have additional filter + metadata defined in a new 'filters' field, and an optional 'filtered\_out' + field per tor circuit # Changes in version 0.6 - 2020-08-08 diff --git a/README.md b/README.md index 224972c..13a9eee 100644 --- a/README.md +++ b/README.md @@ -254,10 +254,12 @@ onionperf analyze --help ``` ### Filtering measurement results -OnionPerf measurement results can be filtered based on Tor relay fingerprints. -The `filter` mode takes a list of fingerprints and one or more existing analysis files as inputs, and outputs new analysis files containing only the `tgen` results obtained over a Tor circuit path which includes or excludes fingerprints in the input list. +The `filter` subcommand is typically used in combination with the `visualize` subcommand. The workflow is to filter out any Tor streams/circuits that are not desired then visualize only those measurements with an existing mapping between TGen transfers/streams and Tor streams/circuits. -Where excluding fingerprints, if ANY relay fingerprint is matched, the measurement is discarded. Where including fingerprints, ALL relay fingerprints in a path must match for the measurement to be retained. +Currently, OnionPerf measurement results can be filtered based on Tor relay fingerprints, although support for filtering TGen based on transfers/streams may be added in the future. + +The `filter` mode takes a list of fingerprints and one or more existing analysis files as inputs, and outputs new analysis files which include, unchanged, the Tor results obtained over a Tor circuit path which includes or excludes fingerprints in the input list. All other Tor results are also included in the file, but are marked as 'filtered\_out'. +Filter metadata detailing the filter type and path to the input list used is also included in the analysis file. For example, the analysis file produced above can be filtered with the following command, which retains measurements based on fingerprints contained in the file 'fingerprints.txt': @@ -290,6 +292,8 @@ As a result, two files are written to the current working directory: - `onionperf.viz.$datetime.csv` contains visualized data in a CSV file format; and - `onionperf.viz.$datetime.pdf` contains visualizations in a PDF file format. +For analysis files containing tor circuit filters, only measurements with an existing mapping between TGen transfers/streams Tor streams/circuits which have not been marked as 'filtered\_out' are visualized. + Similar to the other modes, OnionPerf's `visualize` mode has command-line parameters for customizing the visualization step: ```shell ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/master] Order analysis and filter keys, authored by Karsten
commit 7b0c91f1983ab21f39f37b83d3a58db5a4d11a8f Author: Ana Custura Date: Thu Sep 10 12:37:00 2020 +0100 Order analysis and filter keys, authored by Karsten --- onionperf/filtering.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/onionperf/filtering.py b/onionperf/filtering.py index c008c03..7ef6168 100644 --- a/onionperf/filtering.py +++ b/onionperf/filtering.py @@ -65,10 +65,12 @@ class Filtering(object): break if not keep: tor_circuits[circuit_id]["filtered"] = True +tor_circuits[circuit_id] = dict(sorted(tor_circuit.items())) def apply_filters(self, input_path, output_dir, output_file): self.analysis = OPAnalysis.load(filename=input_path) self.filter_tor_circuits(self.analysis) self.analysis.json_db["filters"] = self.filters +self.analysis.json_db = dict(sorted(self.analysis.json_db.items())) self.analysis.save(filename=output_file, output_prefix=output_dir, sort_keys=False) ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/master] Bump analysis version to 4.0
commit d0660c1847b46863783b7006efdac4df8103f123
Author: Ana Custura
Date: Thu Sep 10 12:54:41 2020 +0100
Bump analysis version to 4.0
---
onionperf/analysis.py | 4 ++--
onionperf/filtering.py | 1 +
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/onionperf/analysis.py b/onionperf/analysis.py
index 49c109c..907d451 100644
--- a/onionperf/analysis.py
+++ b/onionperf/analysis.py
@@ -24,7 +24,7 @@ class OPAnalysis(Analysis):
def __init__(self, nickname=None, ip_address=None):
super().__init__(nickname, ip_address)
-self.json_db = {'type': 'onionperf', 'version': '3.0', 'data': {}}
+self.json_db = {'type': 'onionperf', 'version': '4.0', 'data': {}}
self.torctl_filepaths = []
def add_torctl_file(self, filepath):
@@ -133,7 +133,7 @@ class OPAnalysis(Analysis):
if 'type' not in db or 'version' not in db:
logging.warning("'type' or 'version' not present in database")
return None
-elif db['type'] != 'onionperf' or str(db['version']) >= '4.':
+elif db['type'] != 'onionperf' or str(db['version']) >= '5.':
logging.warning("type or version not supported (type={0},
version={1})".format(db['type'], db['version']))
return None
else:
diff --git a/onionperf/filtering.py b/onionperf/filtering.py
index 0ba9ebc..e8ea164 100644
--- a/onionperf/filtering.py
+++ b/onionperf/filtering.py
@@ -71,6 +71,7 @@ class Filtering(object):
self.analysis = OPAnalysis.load(filename=input_path)
self.filter_tor_circuits(self.analysis)
self.analysis.json_db["filters"] = self.filters
+self.analysis.json_db["version"] = '4.0'
self.analysis.json_db = dict(sorted(self.analysis.json_db.items()))
self.analysis.save(filename=output_file, output_prefix=output_dir,
sort_keys=False)
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/master] Merge branch 'acute-task-33260-4' into develop
commit 76517ac669f3e7575965662507ab970b41395324 Merge: c8275b2 f063b63 Author: Karsten Loesing Date: Wed Sep 16 12:28:19 2020 +0200 Merge branch 'acute-task-33260-4' into develop CHANGELOG.md | 9 + README.md | 23 ++ onionperf/analysis.py | 18 +- onionperf/filtering.py | 75 ++ onionperf/onionperf| 73 ++ onionperf/visualization.py | 31 ++- schema/onionperf-4.0.json | 577 + 7 files changed, 793 insertions(+), 13 deletions(-) diff --cc CHANGELOG.md index 80fb14d,13fb202..2c1f5ca --- a/CHANGELOG.md +++ b/CHANGELOG.md @@@ -1,13 -1,12 +1,22 @@@ + # Changes in version 0.8 - 2020-09-16 + + - Add a new `onionperf filter` mode that takes an OnionPerf analysis +results file or directory as input, applies filters, and produces +new OnionPerf analysis results file(s) as output. Bump the analysis +version number to 4.0 to include additional filter metadata defined +in a 'filters' field and an optional 'filtered\_out' field per Tor +circuit. Implements #33260. + +# Changes in version 0.7 - 2020-09-01 + + - Add `onionperf measure --drop-guards` parameter to use and drop + guards and circuit build timeouts every given number of hours, if + supported by the Tor version. Implements #33399. + - Remove the `onionperf measure --oneshot` switch and replace it with + new switches `--tgen-pause-initial`, `--tgen-pause-between`, + `--tgen-transfer-size`, and `--tgen-num-transfers ` to further + configure the generated TGen model. Implemets #33432. + # Changes in version 0.6 - 2020-08-08 - Update to TGen 1.0.0, use TGenTools for parsing TGen log files, and ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/master] Add JSON schema for analysis version 4.0.
commit a4a4dcc97d07b2668fbab593165119bd52f694ed
Author: Karsten Loesing
Date: Wed Sep 16 11:19:58 2020 +0200
Add JSON schema for analysis version 4.0.
---
schema/onionperf-4.0.json | 577 ++
1 file changed, 577 insertions(+)
diff --git a/schema/onionperf-4.0.json b/schema/onionperf-4.0.json
new file mode 100644
index 000..3c4dba3
--- /dev/null
+++ b/schema/onionperf-4.0.json
@@ -0,0 +1,577 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema;,
+ "$id":
"https://gitlab.torproject.org/tpo/metrics/onionperf/-/raw/master/schema/onionperf-4.0.json;,
+ "type": "object",
+ "title": "OnionPerf analysis JSON file format 4.0",
+ "required": [
+"data",
+"type",
+"version"
+ ],
+ "properties": {
+"data": {
+ "type": "object",
+ "title": "Measurement data by source name",
+ "propertyNames": {
+"pattern": "^[A-Za-z0-9-]+$"
+ },
+ "additionalProperties": {
+"type": "object",
+"title": "Measurement data from a single source",
+"required": [
+ "measurement_ip",
+ "tgen",
+ "tor"
+],
+"properties": {
+ "measurement_ip": {
+"type": "string",
+"title": "Public IP address of the measuring host."
+ },
+ "tgen": {
+"type": "object",
+"title": "Measurement data obtained from client-side TGen logs",
+"required": [
+ "streams"
+],
+"properties": {
+ "streams": {
+"type": "object",
+"title": "Measurement data, by TGen stream identifier",
+"additionalProperties": {
+ "type": "object",
+ "title": "Information on a single measurement, obtained from
a single [stream-success] or [stream-error] log message (except for
elapsed_seconds)",
+ "required": [
+"byte_info",
+"is_complete",
+"is_error",
+"is_success",
+"stream_id",
+"stream_info",
+"time_info",
+"transport_info",
+"unix_ts_end",
+"unix_ts_start"
+ ],
+ "properties": {
+"byte_info": {
+ "type": "object",
+ "title": "Information on sent and received bytes",
+ "required": [
+"payload-bytes-recv",
+"payload-bytes-send",
+"payload-progress-recv",
+"payload-progress-send",
+"total-bytes-recv",
+"total-bytes-send"
+ ],
+ "properties": {
+"payload-bytes-recv": {
+ "type": "string",
+ "pattern": "^[0-9]+$",
+ "title": "Number of payload bytes received"
+},
+"payload-bytes-send": {
+ "type": "string",
+ "pattern": "^[0-9]+$",
+ "title": "Number of payload bytes sent"
+},
+"payload-progress-recv": {
+ "type": "string",
+ "pattern": "^[0-9]+\\.[0-9]+%$",
+ "title": "Progress of receiving payload in percent"
+},
+"payload-progress-send": {
+ "type": "string",
+ "pattern": "^[0-9]+\\.[0-9]+%$",
+ "title": "Progress of sending payload in percent"
+},
+
[tor-commits] [onionperf/master] Add filter section to README.md
commit 52506522bc6dc679318e2af801f7c3a94b6947d0 Author: Ana Custura Date: Sat Aug 29 17:01:13 2020 +0100 Add filter section to README.md --- README.md | 23 +++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index ad53a9e..224972c 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ + [Troubleshooting](#troubleshooting) * [Analysis](#analysis) + [Analyzing measurement results](#analyzing-measurement-results) ++ [Filtering measurement results](#filtering-measurement-results) + [Visualizing measurement results](#visualizing-measurement-results) + [Interpreting the PDF output format](#interpreting-the-pdf-output-format) + [Interpreting the CSV output format](#interpreting-the-csv-output-format) @@ -251,6 +252,28 @@ OnionPerf's `analyze` mode has several command-line parameters for customizing t ```shell onionperf analyze --help ``` +### Filtering measurement results + +OnionPerf measurement results can be filtered based on Tor relay fingerprints. +The `filter` mode takes a list of fingerprints and one or more existing analysis files as inputs, and outputs new analysis files containing only the `tgen` results obtained over a Tor circuit path which includes or excludes fingerprints in the input list. + +Where excluding fingerprints, if ANY relay fingerprint is matched, the measurement is discarded. Where including fingerprints, ALL relay fingerprints in a path must match for the measurement to be retained. + +For example, the analysis file produced above can be filtered with the following command, which retains measurements based on fingerprints contained in the file 'fingerprints.txt': + +```shell +onionperf filter -i onionperf.analysis.json.xz -o filtered.onionperf.analysis.json.xz --include-fingerprints fingerprints.txt +``` + +The output analysis file is written to the path specified with `-o`. If processing a directory of analysis files, its structure and filenames are preserved under the path specified with '-o'. +Note that while the subcommand filters `tgen` measurements, it leaves `tgen` and `tor` summaries in the original analysis file unchanged. + +OnionPerf's `filter` command usage can be inspected with: + +```shell +onionperf filter --help +``` + ### Visualizing measurement results ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/master] Bump version to 0.8.
commit 118d14990ce4821232ed6751c6a3d2b05e872dbe
Author: Karsten Loesing
Date: Wed Sep 16 12:29:43 2020 +0200
Bump version to 0.8.
---
setup.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/setup.py b/setup.py
index d72b27a..b7b00d0 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@ with open('requirements.txt') as f:
install_requires = f.readlines()
setup(name='OnionPerf',
- version='0.7',
+ version='0.8',
description='A utility to monitor, measure, analyze, and visualize the
performance of Tor and Onion Services',
author='Rob Jansen',
url='https://github.com/robgjansen/onionperf/',
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/master] Clarify filter terminology
commit a466221c5b947bb894a36fd3e5472397f1b0b03e Author: Ana Custura Date: Thu Sep 10 12:41:46 2020 +0100 Clarify filter terminology --- onionperf/filtering.py | 2 +- onionperf/visualization.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/onionperf/filtering.py b/onionperf/filtering.py index 7ef6168..0ba9ebc 100644 --- a/onionperf/filtering.py +++ b/onionperf/filtering.py @@ -64,7 +64,7 @@ class Filtering(object): keep = False break if not keep: -tor_circuits[circuit_id]["filtered"] = True +tor_circuits[circuit_id]["filtered_out"] = True tor_circuits[circuit_id] = dict(sorted(tor_circuit.items())) def apply_filters(self, input_path, output_dir, output_file): diff --git a/onionperf/visualization.py b/onionperf/visualization.py index f5bc03f..2cd3161 100644 --- a/onionperf/visualization.py +++ b/onionperf/visualization.py @@ -147,7 +147,7 @@ class TGenVisualization(Visualization): stream["error_code"] = "/".join(error_code_parts) if "filters" in analysis.json_db.keys() and analysis.json_db["filters"]["tor/circuits"]: - if tor_circuit and "filtered" not in tor_circuit.keys(): + if tor_circuit and "filtered_out" not in tor_circuit.keys(): streams.append(stream) else: streams.append(stream) ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/master] Attempt to clean up CHANGELOG.md and README.md.
commit f063b636fd26a938a52508424d7abe62cd45ccf0 Author: Karsten Loesing Date: Wed Sep 16 12:26:56 2020 +0200 Attempt to clean up CHANGELOG.md and README.md. --- CHANGELOG.md | 12 +--- README.md| 14 +- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7d5005..13fb202 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,11 @@ -# Changes in version 0.7 - 2020-??-?? +# Changes in version 0.8 - 2020-09-16 - Add a new `onionperf filter` mode that takes an OnionPerf analysis results file or directory as input, applies filters, and produces - new OnionPerf analysis results file(s) as output. - - Bump the analysis version number to 4.0 for new analyses and analyses - produced by the `onionperf filter` mode - - Analyses produced by the `onionperf filter` mode have additional filter - metadata defined in a new 'filters' field, and an optional 'filtered\_out' - field per tor circuit + new OnionPerf analysis results file(s) as output. Bump the analysis + version number to 4.0 to include additional filter metadata defined + in a 'filters' field and an optional 'filtered\_out' field per Tor + circuit. Implements #33260. # Changes in version 0.6 - 2020-08-08 diff --git a/README.md b/README.md index 13a9eee..6e7684f 100644 --- a/README.md +++ b/README.md @@ -252,31 +252,27 @@ OnionPerf's `analyze` mode has several command-line parameters for customizing t ```shell onionperf analyze --help ``` + ### Filtering measurement results -The `filter` subcommand is typically used in combination with the `visualize` subcommand. The workflow is to filter out any Tor streams/circuits that are not desired then visualize only those measurements with an existing mapping between TGen transfers/streams and Tor streams/circuits. +The `filter` subcommand can be used to filter out measurement results based on given criteria. This subcommand is typically used in combination with the `visualize` subcommand. The workflow is to apply one or more filters and then visualize only those measurements with an existing mapping between TGen transfers/streams and Tor streams/circuits. -Currently, OnionPerf measurement results can be filtered based on Tor relay fingerprints, although support for filtering TGen based on transfers/streams may be added in the future. +Currently, OnionPerf measurement results can be filtered based on Tor relay fingerprints found in Tor circuits, although support for filtering based on Tor streams and/or TGen transfers/streams may be added in the future. -The `filter` mode takes a list of fingerprints and one or more existing analysis files as inputs, and outputs new analysis files which include, unchanged, the Tor results obtained over a Tor circuit path which includes or excludes fingerprints in the input list. All other Tor results are also included in the file, but are marked as 'filtered\_out'. -Filter metadata detailing the filter type and path to the input list used is also included in the analysis file. +The `filter` mode takes a list of fingerprints and one or more existing analysis files as inputs and outputs new analysis files with the same contents as the input analysis files plus annotations on those Tor circuits that have been filtered out. If a directory of analysis files is given to '-i', the structure and filenames of that directory are preserved under the path specified with '-o'. -For example, the analysis file produced above can be filtered with the following command, which retains measurements based on fingerprints contained in the file 'fingerprints.txt': +For example, the analysis file produced above can be filtered with the following command, which retains only those Tor circuits with fingerprints contained in the file 'fingerprints.txt': ```shell onionperf filter -i onionperf.analysis.json.xz -o filtered.onionperf.analysis.json.xz --include-fingerprints fingerprints.txt ``` -The output analysis file is written to the path specified with `-o`. If processing a directory of analysis files, its structure and filenames are preserved under the path specified with '-o'. -Note that while the subcommand filters `tgen` measurements, it leaves `tgen` and `tor` summaries in the original analysis file unchanged. - OnionPerf's `filter` command usage can be inspected with: ```shell onionperf filter --help ``` - ### Visualizing measurement results Step two in the analysis is to process analysis files with OnionPerf's `visualize` mode which produces CSV and PDF files as output. ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/master] Fix naming of exclusion list
commit d9f8b8fe36fabb30bf96802130a68b9020f6a875 Author: Ana Custura Date: Fri Aug 21 03:07:30 2020 +0100 Fix naming of exclusion list --- onionperf/filtering.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onionperf/filtering.py b/onionperf/filtering.py index 204b161..ab96f1e 100644 --- a/onionperf/filtering.py +++ b/onionperf/filtering.py @@ -28,13 +28,13 @@ class Filtering(object): self.fingerprints_to_include.append(fingerprint) def exclude_fingerprints(self, path): -self.exclude_fingerprints = [] +self.fingerprints_to_exclude = [] with open(path, 'rt') as f: for line in f: fingerprint_match = self.fingerprint_pattern.match(line) if fingerprint_match: fingerprint = fingerprint_match.group(1).upper() -self.exclude_fingerprints.append(fingerprint) +self.fingerprints_to_exclude.append(fingerprint) def apply_filters(self): if self.fingerprints_to_include is None and self.fingerprints_to_exclude is None: ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/master] Prevent incorrectly keeping transfers for empty incl./excl. lists
commit efe2ef11eb91e803e44229f5ede0209ead0aae21 Author: Ana Custura Date: Sat Aug 29 14:18:01 2020 +0100 Prevent incorrectly keeping transfers for empty incl./excl. lists --- onionperf/filtering.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onionperf/filtering.py b/onionperf/filtering.py index b431eb9..9e7b34f 100644 --- a/onionperf/filtering.py +++ b/onionperf/filtering.py @@ -79,10 +79,10 @@ class Filtering(object): fingerprint_match = self.fingerprint_pattern.match(long_name) if fingerprint_match: fingerprint = fingerprint_match.group(1).upper() -if self.fingerprints_to_include and fingerprint not in self.fingerprints_to_include: +if self.fingerprints_to_include is not None and fingerprint not in self.fingerprints_to_include: keep = False break -if self.fingerprints_to_exclude and fingerprint in self.fingerprints_to_exclude: +if self.fingerprints_to_exclude is not None and fingerprint in self.fingerprints_to_exclude: keep = False break if keep: ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/master] Add new filter mode to filter analysis results.
commit dcab214322356db9d975673d19146ff46ec86b4f
Author: Karsten Loesing
Date: Tue Aug 18 21:05:45 2020 +0200
Add new filter mode to filter analysis results.
Implements tpo/metrics/onionperf#33260.
---
CHANGELOG.md | 6 +++
onionperf/analysis.py | 11 ++
onionperf/filtering.py | 100 +
onionperf/onionperf| 51 +
4 files changed, 168 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b8a86ee..c57695e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,9 @@
+# Changes in version 0.7 - 2020-??-??
+
+ - Add a new `onionperf filter` mode that takes an OnionPerf analysis
+ results file as input, applies filters, and produces a new
+ OnionPerf analysis results file as output.
+
# Changes in version 0.6 - 2020-08-08
- Update to TGen 1.0.0, use TGenTools for parsing TGen log files, and
diff --git a/onionperf/analysis.py b/onionperf/analysis.py
index ea07d0e..b2f483f 100644
--- a/onionperf/analysis.py
+++ b/onionperf/analysis.py
@@ -62,6 +62,11 @@ class OPAnalysis(Analysis):
self.json_db['data'][self.nickname]["tgen"].pop("stream_summary")
self.did_analysis = True
+def set_tgen_transfers(self, node, tgen_transfers):
+self.json_db['data'][node]['tgen']['transfers'] = tgen_transfers
+
+def set_tgen_streams(self, node, tgen_streams):
+self.json_db['data'][node]['tgen']['streams'] = tgen_streams
def save(self, filename=None, output_prefix=os.getcwd(), do_compress=True,
date_prefix=None):
if filename is None:
@@ -98,6 +103,12 @@ class OPAnalysis(Analysis):
except:
return None
+def get_tor_circuits(self, node):
+try:
+return self.json_db['data'][node]['tor']['circuits']
+except:
+return None
+
def get_tor_streams(self, node):
try:
return self.json_db['data'][node]['tor']['streams']
diff --git a/onionperf/filtering.py b/onionperf/filtering.py
new file mode 100644
index 000..435a1bc
--- /dev/null
+++ b/onionperf/filtering.py
@@ -0,0 +1,100 @@
+'''
+ OnionPerf
+ Authored by Rob Jansen, 2015
+ Copyright 2015-2020 The Tor Project
+ See LICENSE for licensing information
+'''
+
+import re
+from onionperf.analysis import OPAnalysis
+
+class Filtering(object):
+
+def __init__(self):
+self.fingerprints_to_include = None
+self.fingerprints_to_exclude = None
+self.fingerprint_pattern = re.compile("\$?([0-9a-fA-F]{40})")
+
+def read_input(self, path):
+self.analysis = OPAnalysis.load(filename=path)
+
+def include_fingerprints(self, path):
+self.fingerprints_to_include = []
+with open(path, 'rt') as f:
+for line in f:
+fingerprint_match = self.fingerprint_pattern.match(line)
+if fingerprint_match:
+fingerprint = fingerprint_match.group(1).upper()
+self.fingerprints_to_include.append(fingerprint)
+
+def exclude_fingerprints(self, path):
+self.exclude_fingerprints = []
+with open(path, 'rt') as f:
+for line in f:
+fingerprint_match = self.fingerprint_pattern.match(line)
+if fingerprint_match:
+fingerprint = fingerprint_match.group(1).upper()
+self.exclude_fingerprints.append(fingerprint)
+
+def apply_filters(self):
+if self.fingerprints_to_include is None and
self.fingerprints_to_exclude is None:
+return
+for source in self.analysis.get_nodes():
+tor_streams_by_source_port = {}
+tor_streams = self.analysis.get_tor_streams(source)
+for tor_stream in tor_streams.values():
+if "source" in tor_stream and ":" in tor_stream["source"]:
+source_port = tor_stream["source"].split(":")[1]
+tor_streams_by_source_port.setdefault(source_port,
[]).append(tor_stream)
+tor_circuits = self.analysis.get_tor_circuits(source)
+tgen_streams = self.analysis.get_tgen_streams(source)
+tgen_transfers = self.analysis.get_tgen_transfers(source)
+retained_tgen_streams = {}
+retained_tgen_transfers = {}
+while tgen_streams or tgen_transfers:
+stream_id = None
+transfer_id = None
+source_port = None
+unix_ts_end = None
+keep = False
+if tgen_streams:
+stream_id, stream_data = tgen_streams.popitem()
+if "local" in stream_data["transport_info"] and
len(stream_data["transport_info"]["local"].split(":")) > 2:
+source_port =
stream_data["transp
[tor-commits] [onionperf/develop] Bump version to 0.8.
commit 118d14990ce4821232ed6751c6a3d2b05e872dbe
Author: Karsten Loesing
Date: Wed Sep 16 12:29:43 2020 +0200
Bump version to 0.8.
---
setup.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/setup.py b/setup.py
index d72b27a..b7b00d0 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@ with open('requirements.txt') as f:
install_requires = f.readlines()
setup(name='OnionPerf',
- version='0.7',
+ version='0.8',
description='A utility to monitor, measure, analyze, and visualize the
performance of Tor and Onion Services',
author='Rob Jansen',
url='https://github.com/robgjansen/onionperf/',
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/master] Also accept a directory in `onionperf filter -i`.
commit 053da92c69d4b3628f33e1dd610a1bc8f601cbe0
Author: Karsten Loesing
Date: Thu Aug 27 09:59:59 2020 +0200
Also accept a directory in `onionperf filter -i`.
And clarify that we're leaving statistics unchanged as part of the
filtering.
---
CHANGELOG.md | 4 ++--
onionperf/filtering.py | 10 +++---
onionperf/onionperf| 35 ---
3 files changed, 29 insertions(+), 20 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c57695e..ebd43e1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,8 +1,8 @@
# Changes in version 0.7 - 2020-??-??
- Add a new `onionperf filter` mode that takes an OnionPerf analysis
- results file as input, applies filters, and produces a new
- OnionPerf analysis results file as output.
+ results file or directory as input, applies filters, and produces
+ new OnionPerf analysis results file(s) as output.
# Changes in version 0.6 - 2020-08-08
diff --git a/onionperf/filtering.py b/onionperf/filtering.py
index ab96f1e..b431eb9 100644
--- a/onionperf/filtering.py
+++ b/onionperf/filtering.py
@@ -15,9 +15,6 @@ class Filtering(object):
self.fingerprints_to_exclude = None
self.fingerprint_pattern = re.compile("\$?([0-9a-fA-F]{40})")
-def read_input(self, path):
-self.analysis = OPAnalysis.load(filename=path)
-
def include_fingerprints(self, path):
self.fingerprints_to_include = []
with open(path, 'rt') as f:
@@ -36,7 +33,8 @@ class Filtering(object):
fingerprint = fingerprint_match.group(1).upper()
self.fingerprints_to_exclude.append(fingerprint)
-def apply_filters(self):
+def apply_filters(self, input_path, output_dir, output_file):
+self.analysis = OPAnalysis.load(filename=input_path)
if self.fingerprints_to_include is None and
self.fingerprints_to_exclude is None:
return
for source in self.analysis.get_nodes():
@@ -94,7 +92,5 @@ class Filtering(object):
retained_tgen_transfers[transfer_id] = transfer_data
self.analysis.set_tgen_streams(source, retained_tgen_streams)
self.analysis.set_tgen_transfers(source, retained_tgen_transfers)
-
-def write_output(self, path):
-self.analysis.save(filename=path)
+self.analysis.save(filename=output_file, output_prefix=output_dir)
diff --git a/onionperf/onionperf b/onionperf/onionperf
index 96c6869..7c16aea 100755
--- a/onionperf/onionperf
+++ b/onionperf/onionperf
@@ -76,8 +76,11 @@ Analyze Tor and TGen output
"""
DESC_FILTER = """
-Takes an OnionPerf analysis results file as input, applies filters,
-and produces a new OnionPerf analysis results file as output.
+Takes an OnionPerf analysis results file or directory as input, applies
filters,
+and produces new OnionPerf analysis results file(s) as output.
+
+This subcommand only filters measurements in `data/[source]/tgen/transfers`
+and `data/[source]/tgen/streams`, but leaves any summaries unchanged.
"""
HELP_FILTER = """
Filter OnionPerf analysis results
@@ -295,7 +298,8 @@ files generated by this script will be written""",
filter_parser.set_defaults(func=filter, formatter_class=my_formatter_class)
filter_parser.add_argument('-i', '--input',
-help="""read the OnionPerf analysis results at PATH as input""",
+help="""a file or directory PATH from which OnionPerf analysis results
+files are read""",
metavar="PATH", required="True",
action="store", dest="input")
@@ -314,8 +318,8 @@ files generated by this script will be written""",
default=None)
filter_parser.add_argument('-o', '--output',
-help="""write the filtered output OnionPerf analysis results file to
-PATH""",
+help="""a file or directory PATH where filtered output OnionPerf
+analysis results files are written""",
metavar="PATH", required="True",
action="store", dest="output")
@@ -439,17 +443,26 @@ def analyze(args):
def filter(args):
from onionperf.filtering import Filtering
-p = os.path.abspath(os.path.expanduser(args.input))
-if not os.path.exists(p):
-raise argparse.ArgumentTypeError("path '%s' does not exist" %
args.input)
+input_path = os.path.abspath(os.path.expanduser(args.input))
+if not os.path.exists(input_path):
+raise argparse.ArgumentTypeError("input path '%s' does not exist" %
args.input)
+output_path = os.path.abspath(os.path.expanduser(args.output))
+if os.path.exists(output_path)
[tor-commits] [onionperf/master] Fix matching of tor_circuit ids
commit 26ff79a3e57f1dfb0c961cceb167534bb39d17f3 Author: Ana Custura Date: Fri Aug 21 02:14:32 2020 +0100 Fix matching of tor_circuit ids --- onionperf/filtering.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onionperf/filtering.py b/onionperf/filtering.py index 435a1bc..204b161 100644 --- a/onionperf/filtering.py +++ b/onionperf/filtering.py @@ -73,7 +73,7 @@ class Filtering(object): for tor_stream in tor_streams_by_source_port[source_port]: if abs(unix_ts_end - tor_stream["unix_ts_end"]) < 150.0: circuit_id = tor_stream["circuit_id"] -if circuit_id and circuit_id in tor_circuits: +if circuit_id and str(circuit_id) in tor_circuits: tor_circuit = tor_circuits[circuit_id] path = tor_circuit["path"] keep = True ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/master] Add path validation to filter interface
commit 1a81b3dff756cf197d661f931485443d1454c7c4
Author: Ana Custura
Date: Fri Aug 21 01:38:01 2020 +0100
Add path validation to filter interface
---
onionperf/onionperf | 3 +++
1 file changed, 3 insertions(+)
diff --git a/onionperf/onionperf b/onionperf/onionperf
index b1e7bd3..96c6869 100755
--- a/onionperf/onionperf
+++ b/onionperf/onionperf
@@ -439,6 +439,9 @@ def analyze(args):
def filter(args):
from onionperf.filtering import Filtering
+p = os.path.abspath(os.path.expanduser(args.input))
+if not os.path.exists(p):
+raise argparse.ArgumentTypeError("path '%s' does not exist" %
args.input)
filtering = Filtering()
filtering.read_input(args.input)
if args.include_fingerprints is not None:
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/develop] Change filter mode to filter Tor circuits.
commit 9d0c80561b84905e72b41758dfcb7b712f5f407f
Author: Karsten Loesing
Date: Mon Aug 31 11:30:53 2020 +0200
Change filter mode to filter Tor circuits.
This new filter mode removes Tor circuits that don't match the
provided fingerprints and leaves TGen transfers/streams untouched. At
the same time the visualize mode only includes TGen transfers/streams
with an existing mapping between TGen transfers/streams and Tor
streams/circuits.
This patch changes the default behavior of the visualize mode. The
original behavior of visualizing TGen transfers/streams *without* an
existing mapping to Tor streams/circuits can be selected with the
--outer-join switch, even though that's rather an edge use case.
Another minor change is that the filtered analysis files is not
written with sort_keys=True anymore, which would have produced a newly
sorted file with keys in alphabetic order rather than the original
insert order. The result is an actually useful diff.
---
onionperf/analysis.py | 13 +---
onionperf/filtering.py | 49 --
onionperf/onionperf| 24 +--
onionperf/visualization.py | 33 ---
4 files changed, 53 insertions(+), 66 deletions(-)
diff --git a/onionperf/analysis.py b/onionperf/analysis.py
index b2f483f..49c109c 100644
--- a/onionperf/analysis.py
+++ b/onionperf/analysis.py
@@ -62,13 +62,7 @@ class OPAnalysis(Analysis):
self.json_db['data'][self.nickname]["tgen"].pop("stream_summary")
self.did_analysis = True
-def set_tgen_transfers(self, node, tgen_transfers):
-self.json_db['data'][node]['tgen']['transfers'] = tgen_transfers
-
-def set_tgen_streams(self, node, tgen_streams):
-self.json_db['data'][node]['tgen']['streams'] = tgen_streams
-
-def save(self, filename=None, output_prefix=os.getcwd(), do_compress=True,
date_prefix=None):
+def save(self, filename=None, output_prefix=os.getcwd(), do_compress=True,
date_prefix=None, sort_keys=True):
if filename is None:
base_filename = "onionperf.analysis.json.xz"
if date_prefix is not None:
@@ -85,7 +79,7 @@ class OPAnalysis(Analysis):
logging.info("saving analysis results to {0}".format(filepath))
outf = util.FileWritable(filepath, do_compress=do_compress)
-json.dump(self.json_db, outf, sort_keys=True, separators=(',', ': '),
indent=2)
+json.dump(self.json_db, outf, sort_keys=sort_keys, separators=(',', ':
'), indent=2)
outf.close()
logging.info("done!")
@@ -109,6 +103,9 @@ class OPAnalysis(Analysis):
except:
return None
+def set_tor_circuits(self, node, tor_circuits):
+self.json_db['data'][node]['tor']['circuits'] = tor_circuits
+
def get_tor_streams(self, node):
try:
return self.json_db['data'][node]['tor']['streams']
diff --git a/onionperf/filtering.py b/onionperf/filtering.py
index 9e7b34f..1b614d6 100644
--- a/onionperf/filtering.py
+++ b/onionperf/filtering.py
@@ -38,41 +38,11 @@ class Filtering(object):
if self.fingerprints_to_include is None and
self.fingerprints_to_exclude is None:
return
for source in self.analysis.get_nodes():
-tor_streams_by_source_port = {}
-tor_streams = self.analysis.get_tor_streams(source)
-for tor_stream in tor_streams.values():
-if "source" in tor_stream and ":" in tor_stream["source"]:
-source_port = tor_stream["source"].split(":")[1]
-tor_streams_by_source_port.setdefault(source_port,
[]).append(tor_stream)
tor_circuits = self.analysis.get_tor_circuits(source)
-tgen_streams = self.analysis.get_tgen_streams(source)
-tgen_transfers = self.analysis.get_tgen_transfers(source)
-retained_tgen_streams = {}
-retained_tgen_transfers = {}
-while tgen_streams or tgen_transfers:
-stream_id = None
-transfer_id = None
-source_port = None
-unix_ts_end = None
+filtered_circuit_ids = []
+for circuit_id, tor_circuit in tor_circuits.items():
keep = False
-if tgen_streams:
-stream_id, stream_data = tgen_streams.popitem()
-if "local" in stream_data["transport_info"] and
len(stream_data["transport_info"]["local"].split(":")) > 2:
-source_port =
stream_data["transport_info"]["local"].split(":")[2]
-if "unix_ts_end" in stream_data:
-
[tor-commits] [onionperf/develop] Merge branch 'acute-task-33260-4' into develop
commit 76517ac669f3e7575965662507ab970b41395324 Merge: c8275b2 f063b63 Author: Karsten Loesing Date: Wed Sep 16 12:28:19 2020 +0200 Merge branch 'acute-task-33260-4' into develop CHANGELOG.md | 9 + README.md | 23 ++ onionperf/analysis.py | 18 +- onionperf/filtering.py | 75 ++ onionperf/onionperf| 73 ++ onionperf/visualization.py | 31 ++- schema/onionperf-4.0.json | 577 + 7 files changed, 793 insertions(+), 13 deletions(-) diff --cc CHANGELOG.md index 80fb14d,13fb202..2c1f5ca --- a/CHANGELOG.md +++ b/CHANGELOG.md @@@ -1,13 -1,12 +1,22 @@@ + # Changes in version 0.8 - 2020-09-16 + + - Add a new `onionperf filter` mode that takes an OnionPerf analysis +results file or directory as input, applies filters, and produces +new OnionPerf analysis results file(s) as output. Bump the analysis +version number to 4.0 to include additional filter metadata defined +in a 'filters' field and an optional 'filtered\_out' field per Tor +circuit. Implements #33260. + +# Changes in version 0.7 - 2020-09-01 + + - Add `onionperf measure --drop-guards` parameter to use and drop + guards and circuit build timeouts every given number of hours, if + supported by the Tor version. Implements #33399. + - Remove the `onionperf measure --oneshot` switch and replace it with + new switches `--tgen-pause-initial`, `--tgen-pause-between`, + `--tgen-transfer-size`, and `--tgen-num-transfers ` to further + configure the generated TGen model. Implemets #33432. + # Changes in version 0.6 - 2020-08-08 - Update to TGen 1.0.0, use TGenTools for parsing TGen log files, and ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/develop] Order analysis and filter keys, authored by Karsten
commit 7b0c91f1983ab21f39f37b83d3a58db5a4d11a8f Author: Ana Custura Date: Thu Sep 10 12:37:00 2020 +0100 Order analysis and filter keys, authored by Karsten --- onionperf/filtering.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/onionperf/filtering.py b/onionperf/filtering.py index c008c03..7ef6168 100644 --- a/onionperf/filtering.py +++ b/onionperf/filtering.py @@ -65,10 +65,12 @@ class Filtering(object): break if not keep: tor_circuits[circuit_id]["filtered"] = True +tor_circuits[circuit_id] = dict(sorted(tor_circuit.items())) def apply_filters(self, input_path, output_dir, output_file): self.analysis = OPAnalysis.load(filename=input_path) self.filter_tor_circuits(self.analysis) self.analysis.json_db["filters"] = self.filters +self.analysis.json_db = dict(sorted(self.analysis.json_db.items())) self.analysis.save(filename=output_file, output_prefix=output_dir, sort_keys=False) ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/develop] Update README and CHANGELOG
commit 1b638e34cb0476148dcf93b43b58b54a30c27428 Author: Ana Custura Date: Thu Sep 10 14:46:10 2020 +0100 Update README and CHANGELOG --- CHANGELOG.md | 5 + README.md| 10 +++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebd43e1..a7d5005 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,11 @@ - Add a new `onionperf filter` mode that takes an OnionPerf analysis results file or directory as input, applies filters, and produces new OnionPerf analysis results file(s) as output. + - Bump the analysis version number to 4.0 for new analyses and analyses + produced by the `onionperf filter` mode + - Analyses produced by the `onionperf filter` mode have additional filter + metadata defined in a new 'filters' field, and an optional 'filtered\_out' + field per tor circuit # Changes in version 0.6 - 2020-08-08 diff --git a/README.md b/README.md index 224972c..13a9eee 100644 --- a/README.md +++ b/README.md @@ -254,10 +254,12 @@ onionperf analyze --help ``` ### Filtering measurement results -OnionPerf measurement results can be filtered based on Tor relay fingerprints. -The `filter` mode takes a list of fingerprints and one or more existing analysis files as inputs, and outputs new analysis files containing only the `tgen` results obtained over a Tor circuit path which includes or excludes fingerprints in the input list. +The `filter` subcommand is typically used in combination with the `visualize` subcommand. The workflow is to filter out any Tor streams/circuits that are not desired then visualize only those measurements with an existing mapping between TGen transfers/streams and Tor streams/circuits. -Where excluding fingerprints, if ANY relay fingerprint is matched, the measurement is discarded. Where including fingerprints, ALL relay fingerprints in a path must match for the measurement to be retained. +Currently, OnionPerf measurement results can be filtered based on Tor relay fingerprints, although support for filtering TGen based on transfers/streams may be added in the future. + +The `filter` mode takes a list of fingerprints and one or more existing analysis files as inputs, and outputs new analysis files which include, unchanged, the Tor results obtained over a Tor circuit path which includes or excludes fingerprints in the input list. All other Tor results are also included in the file, but are marked as 'filtered\_out'. +Filter metadata detailing the filter type and path to the input list used is also included in the analysis file. For example, the analysis file produced above can be filtered with the following command, which retains measurements based on fingerprints contained in the file 'fingerprints.txt': @@ -290,6 +292,8 @@ As a result, two files are written to the current working directory: - `onionperf.viz.$datetime.csv` contains visualized data in a CSV file format; and - `onionperf.viz.$datetime.pdf` contains visualizations in a PDF file format. +For analysis files containing tor circuit filters, only measurements with an existing mapping between TGen transfers/streams Tor streams/circuits which have not been marked as 'filtered\_out' are visualized. + Similar to the other modes, OnionPerf's `visualize` mode has command-line parameters for customizing the visualization step: ```shell ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/develop] Attempt to clean up CHANGELOG.md and README.md.
commit f063b636fd26a938a52508424d7abe62cd45ccf0 Author: Karsten Loesing Date: Wed Sep 16 12:26:56 2020 +0200 Attempt to clean up CHANGELOG.md and README.md. --- CHANGELOG.md | 12 +--- README.md| 14 +- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7d5005..13fb202 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,11 @@ -# Changes in version 0.7 - 2020-??-?? +# Changes in version 0.8 - 2020-09-16 - Add a new `onionperf filter` mode that takes an OnionPerf analysis results file or directory as input, applies filters, and produces - new OnionPerf analysis results file(s) as output. - - Bump the analysis version number to 4.0 for new analyses and analyses - produced by the `onionperf filter` mode - - Analyses produced by the `onionperf filter` mode have additional filter - metadata defined in a new 'filters' field, and an optional 'filtered\_out' - field per tor circuit + new OnionPerf analysis results file(s) as output. Bump the analysis + version number to 4.0 to include additional filter metadata defined + in a 'filters' field and an optional 'filtered\_out' field per Tor + circuit. Implements #33260. # Changes in version 0.6 - 2020-08-08 diff --git a/README.md b/README.md index 13a9eee..6e7684f 100644 --- a/README.md +++ b/README.md @@ -252,31 +252,27 @@ OnionPerf's `analyze` mode has several command-line parameters for customizing t ```shell onionperf analyze --help ``` + ### Filtering measurement results -The `filter` subcommand is typically used in combination with the `visualize` subcommand. The workflow is to filter out any Tor streams/circuits that are not desired then visualize only those measurements with an existing mapping between TGen transfers/streams and Tor streams/circuits. +The `filter` subcommand can be used to filter out measurement results based on given criteria. This subcommand is typically used in combination with the `visualize` subcommand. The workflow is to apply one or more filters and then visualize only those measurements with an existing mapping between TGen transfers/streams and Tor streams/circuits. -Currently, OnionPerf measurement results can be filtered based on Tor relay fingerprints, although support for filtering TGen based on transfers/streams may be added in the future. +Currently, OnionPerf measurement results can be filtered based on Tor relay fingerprints found in Tor circuits, although support for filtering based on Tor streams and/or TGen transfers/streams may be added in the future. -The `filter` mode takes a list of fingerprints and one or more existing analysis files as inputs, and outputs new analysis files which include, unchanged, the Tor results obtained over a Tor circuit path which includes or excludes fingerprints in the input list. All other Tor results are also included in the file, but are marked as 'filtered\_out'. -Filter metadata detailing the filter type and path to the input list used is also included in the analysis file. +The `filter` mode takes a list of fingerprints and one or more existing analysis files as inputs and outputs new analysis files with the same contents as the input analysis files plus annotations on those Tor circuits that have been filtered out. If a directory of analysis files is given to '-i', the structure and filenames of that directory are preserved under the path specified with '-o'. -For example, the analysis file produced above can be filtered with the following command, which retains measurements based on fingerprints contained in the file 'fingerprints.txt': +For example, the analysis file produced above can be filtered with the following command, which retains only those Tor circuits with fingerprints contained in the file 'fingerprints.txt': ```shell onionperf filter -i onionperf.analysis.json.xz -o filtered.onionperf.analysis.json.xz --include-fingerprints fingerprints.txt ``` -The output analysis file is written to the path specified with `-o`. If processing a directory of analysis files, its structure and filenames are preserved under the path specified with '-o'. -Note that while the subcommand filters `tgen` measurements, it leaves `tgen` and `tor` summaries in the original analysis file unchanged. - OnionPerf's `filter` command usage can be inspected with: ```shell onionperf filter --help ``` - ### Visualizing measurement results Step two in the analysis is to process analysis files with OnionPerf's `visualize` mode which produces CSV and PDF files as output. ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/develop] Add new filter mode to filter analysis results.
commit dcab214322356db9d975673d19146ff46ec86b4f
Author: Karsten Loesing
Date: Tue Aug 18 21:05:45 2020 +0200
Add new filter mode to filter analysis results.
Implements tpo/metrics/onionperf#33260.
---
CHANGELOG.md | 6 +++
onionperf/analysis.py | 11 ++
onionperf/filtering.py | 100 +
onionperf/onionperf| 51 +
4 files changed, 168 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b8a86ee..c57695e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,9 @@
+# Changes in version 0.7 - 2020-??-??
+
+ - Add a new `onionperf filter` mode that takes an OnionPerf analysis
+ results file as input, applies filters, and produces a new
+ OnionPerf analysis results file as output.
+
# Changes in version 0.6 - 2020-08-08
- Update to TGen 1.0.0, use TGenTools for parsing TGen log files, and
diff --git a/onionperf/analysis.py b/onionperf/analysis.py
index ea07d0e..b2f483f 100644
--- a/onionperf/analysis.py
+++ b/onionperf/analysis.py
@@ -62,6 +62,11 @@ class OPAnalysis(Analysis):
self.json_db['data'][self.nickname]["tgen"].pop("stream_summary")
self.did_analysis = True
+def set_tgen_transfers(self, node, tgen_transfers):
+self.json_db['data'][node]['tgen']['transfers'] = tgen_transfers
+
+def set_tgen_streams(self, node, tgen_streams):
+self.json_db['data'][node]['tgen']['streams'] = tgen_streams
def save(self, filename=None, output_prefix=os.getcwd(), do_compress=True,
date_prefix=None):
if filename is None:
@@ -98,6 +103,12 @@ class OPAnalysis(Analysis):
except:
return None
+def get_tor_circuits(self, node):
+try:
+return self.json_db['data'][node]['tor']['circuits']
+except:
+return None
+
def get_tor_streams(self, node):
try:
return self.json_db['data'][node]['tor']['streams']
diff --git a/onionperf/filtering.py b/onionperf/filtering.py
new file mode 100644
index 000..435a1bc
--- /dev/null
+++ b/onionperf/filtering.py
@@ -0,0 +1,100 @@
+'''
+ OnionPerf
+ Authored by Rob Jansen, 2015
+ Copyright 2015-2020 The Tor Project
+ See LICENSE for licensing information
+'''
+
+import re
+from onionperf.analysis import OPAnalysis
+
+class Filtering(object):
+
+def __init__(self):
+self.fingerprints_to_include = None
+self.fingerprints_to_exclude = None
+self.fingerprint_pattern = re.compile("\$?([0-9a-fA-F]{40})")
+
+def read_input(self, path):
+self.analysis = OPAnalysis.load(filename=path)
+
+def include_fingerprints(self, path):
+self.fingerprints_to_include = []
+with open(path, 'rt') as f:
+for line in f:
+fingerprint_match = self.fingerprint_pattern.match(line)
+if fingerprint_match:
+fingerprint = fingerprint_match.group(1).upper()
+self.fingerprints_to_include.append(fingerprint)
+
+def exclude_fingerprints(self, path):
+self.exclude_fingerprints = []
+with open(path, 'rt') as f:
+for line in f:
+fingerprint_match = self.fingerprint_pattern.match(line)
+if fingerprint_match:
+fingerprint = fingerprint_match.group(1).upper()
+self.exclude_fingerprints.append(fingerprint)
+
+def apply_filters(self):
+if self.fingerprints_to_include is None and
self.fingerprints_to_exclude is None:
+return
+for source in self.analysis.get_nodes():
+tor_streams_by_source_port = {}
+tor_streams = self.analysis.get_tor_streams(source)
+for tor_stream in tor_streams.values():
+if "source" in tor_stream and ":" in tor_stream["source"]:
+source_port = tor_stream["source"].split(":")[1]
+tor_streams_by_source_port.setdefault(source_port,
[]).append(tor_stream)
+tor_circuits = self.analysis.get_tor_circuits(source)
+tgen_streams = self.analysis.get_tgen_streams(source)
+tgen_transfers = self.analysis.get_tgen_transfers(source)
+retained_tgen_streams = {}
+retained_tgen_transfers = {}
+while tgen_streams or tgen_transfers:
+stream_id = None
+transfer_id = None
+source_port = None
+unix_ts_end = None
+keep = False
+if tgen_streams:
+stream_id, stream_data = tgen_streams.popitem()
+if "local" in stream_data["transport_info"] and
len(stream_data["transport_info"]["local"].split(":")) > 2:
+source_port =
stream_data["transp
[tor-commits] [onionperf/develop] Move filters and filter metadata to analysis files
commit 95b749a8fc690825c0a828b8473c58faea7ad912
Author: Ana Custura
Date: Thu Sep 10 01:51:54 2020 +0100
Move filters and filter metadata to analysis files
---
onionperf/filtering.py | 25 ++---
onionperf/onionperf| 9 +
onionperf/visualization.py | 14 +-
3 files changed, 28 insertions(+), 20 deletions(-)
diff --git a/onionperf/filtering.py b/onionperf/filtering.py
index 1b614d6..c008c03 100644
--- a/onionperf/filtering.py
+++ b/onionperf/filtering.py
@@ -7,6 +7,7 @@
import re
from onionperf.analysis import OPAnalysis
+from collections import defaultdict
class Filtering(object):
@@ -14,9 +15,11 @@ class Filtering(object):
self.fingerprints_to_include = None
self.fingerprints_to_exclude = None
self.fingerprint_pattern = re.compile("\$?([0-9a-fA-F]{40})")
+self.filters = defaultdict(list)
def include_fingerprints(self, path):
self.fingerprints_to_include = []
+self.fingerprints_to_include_path = path
with open(path, 'rt') as f:
for line in f:
fingerprint_match = self.fingerprint_pattern.match(line)
@@ -26,6 +29,7 @@ class Filtering(object):
def exclude_fingerprints(self, path):
self.fingerprints_to_exclude = []
+self.fingerprints_to_exclude_path = path
with open(path, 'rt') as f:
for line in f:
fingerprint_match = self.fingerprint_pattern.match(line)
@@ -33,12 +37,16 @@ class Filtering(object):
fingerprint = fingerprint_match.group(1).upper()
self.fingerprints_to_exclude.append(fingerprint)
-def apply_filters(self, input_path, output_dir, output_file):
-self.analysis = OPAnalysis.load(filename=input_path)
+def filter_tor_circuits(self, analysis):
if self.fingerprints_to_include is None and
self.fingerprints_to_exclude is None:
return
-for source in self.analysis.get_nodes():
-tor_circuits = self.analysis.get_tor_circuits(source)
+self.filters["tor/circuits"] = []
+if self.fingerprints_to_include:
+ self.filters["tor/circuits"].append({"name":
"include_fingerprints", "filepath": self.fingerprints_to_include_path })
+if self.fingerprints_to_exclude:
+ self.filters["tor/circuits"].append({"name":
"exclude_fingerprints", "filepath": self.fingerprints_to_exclude_path })
+for source in analysis.get_nodes():
+tor_circuits = analysis.get_tor_circuits(source)
filtered_circuit_ids = []
for circuit_id, tor_circuit in tor_circuits.items():
keep = False
@@ -56,8 +64,11 @@ class Filtering(object):
keep = False
break
if not keep:
-filtered_circuit_ids.append(circuit_id)
-for circuit_id in filtered_circuit_ids:
-del(tor_circuits[circuit_id])
+tor_circuits[circuit_id]["filtered"] = True
+
+def apply_filters(self, input_path, output_dir, output_file):
+self.analysis = OPAnalysis.load(filename=input_path)
+self.filter_tor_circuits(self.analysis)
+self.analysis.json_db["filters"] = self.filters
self.analysis.save(filename=output_file, output_prefix=output_dir,
sort_keys=False)
diff --git a/onionperf/onionperf b/onionperf/onionperf
index 1efa8cb..108af4e 100755
--- a/onionperf/onionperf
+++ b/onionperf/onionperf
@@ -342,13 +342,6 @@ files generated by this script will be written""",
required="True",
action=PathStringArgsAction, dest="datasets")
-visualize_parser.add_argument('--outer-join',
-help="""Include measurements without an existing mapping between TGen
-transfers/streams and Tor streams/circuits, which is the
-equivalent of an outer join in the database sense""",
-action="store_true", dest="outer_join",
-default=False)
-
visualize_parser.add_argument('-p', '--prefix',
help="a STRING filename prefix for graphs we generate",
metavar="STRING", type=str,
@@ -489,7 +482,7 @@ def visualize(args):
if analysis is not None:
analyses.append(analysis)
tgen_viz.add_dataset(analyses, label)
-tgen_viz.plot_all(args.prefix, outer_join=args.outer_join)
+tgen_viz.plot_all(args.prefix)
def type_nonnegative_integer(value):
i = int(value)
diff --git a/onionperf/visualization.py b/onionperf/visualization.py
index 0f69879..f5bc03f 100644
--- a/onionperf/visualization.py
+++ b/onionperf/visualization.py
@@ -31,11 +31,11 @@ class Visualization(object, metaclass=ABCMeta):
class TGenVisualization(Visualization):
-def plot_all(self, output_prefix, outer_join=False):
+def plot_all(self, output_prefix):
if len(self.datasets) > 0:
[tor-commits] [onionperf/develop] Add JSON schema for analysis version 4.0.
commit a4a4dcc97d07b2668fbab593165119bd52f694ed
Author: Karsten Loesing
Date: Wed Sep 16 11:19:58 2020 +0200
Add JSON schema for analysis version 4.0.
---
schema/onionperf-4.0.json | 577 ++
1 file changed, 577 insertions(+)
diff --git a/schema/onionperf-4.0.json b/schema/onionperf-4.0.json
new file mode 100644
index 000..3c4dba3
--- /dev/null
+++ b/schema/onionperf-4.0.json
@@ -0,0 +1,577 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema;,
+ "$id":
"https://gitlab.torproject.org/tpo/metrics/onionperf/-/raw/master/schema/onionperf-4.0.json;,
+ "type": "object",
+ "title": "OnionPerf analysis JSON file format 4.0",
+ "required": [
+"data",
+"type",
+"version"
+ ],
+ "properties": {
+"data": {
+ "type": "object",
+ "title": "Measurement data by source name",
+ "propertyNames": {
+"pattern": "^[A-Za-z0-9-]+$"
+ },
+ "additionalProperties": {
+"type": "object",
+"title": "Measurement data from a single source",
+"required": [
+ "measurement_ip",
+ "tgen",
+ "tor"
+],
+"properties": {
+ "measurement_ip": {
+"type": "string",
+"title": "Public IP address of the measuring host."
+ },
+ "tgen": {
+"type": "object",
+"title": "Measurement data obtained from client-side TGen logs",
+"required": [
+ "streams"
+],
+"properties": {
+ "streams": {
+"type": "object",
+"title": "Measurement data, by TGen stream identifier",
+"additionalProperties": {
+ "type": "object",
+ "title": "Information on a single measurement, obtained from
a single [stream-success] or [stream-error] log message (except for
elapsed_seconds)",
+ "required": [
+"byte_info",
+"is_complete",
+"is_error",
+"is_success",
+"stream_id",
+"stream_info",
+"time_info",
+"transport_info",
+"unix_ts_end",
+"unix_ts_start"
+ ],
+ "properties": {
+"byte_info": {
+ "type": "object",
+ "title": "Information on sent and received bytes",
+ "required": [
+"payload-bytes-recv",
+"payload-bytes-send",
+"payload-progress-recv",
+"payload-progress-send",
+"total-bytes-recv",
+"total-bytes-send"
+ ],
+ "properties": {
+"payload-bytes-recv": {
+ "type": "string",
+ "pattern": "^[0-9]+$",
+ "title": "Number of payload bytes received"
+},
+"payload-bytes-send": {
+ "type": "string",
+ "pattern": "^[0-9]+$",
+ "title": "Number of payload bytes sent"
+},
+"payload-progress-recv": {
+ "type": "string",
+ "pattern": "^[0-9]+\\.[0-9]+%$",
+ "title": "Progress of receiving payload in percent"
+},
+"payload-progress-send": {
+ "type": "string",
+ "pattern": "^[0-9]+\\.[0-9]+%$",
+ "title": "Progress of sending payload in percent"
+},
+
[tor-commits] [onionperf/develop] Append to existing "filters" field.
commit c41c0b74c47442dc6f6d23f49a012729f71aee49
Author: Karsten Loesing
Date: Wed Sep 16 12:01:07 2020 +0200
Append to existing "filters" field.
Before this change we would have replaced an existing "filters" field
with the filters applied in this run. But if the user runs `onionperf
filter` more than once to apply different filters, we'll want to list
all filters in the "filters" field.
This change also removes a couple of `self.`s, in particular
`self.analysis` and `self.filters`, because one Filtering object can
apply filters to more than one analysis.
---
onionperf/filtering.py | 20 +---
1 file changed, 9 insertions(+), 11 deletions(-)
diff --git a/onionperf/filtering.py b/onionperf/filtering.py
index e8ea164..15d9c19 100644
--- a/onionperf/filtering.py
+++ b/onionperf/filtering.py
@@ -7,7 +7,6 @@
import re
from onionperf.analysis import OPAnalysis
-from collections import defaultdict
class Filtering(object):
@@ -15,7 +14,6 @@ class Filtering(object):
self.fingerprints_to_include = None
self.fingerprints_to_exclude = None
self.fingerprint_pattern = re.compile("\$?([0-9a-fA-F]{40})")
-self.filters = defaultdict(list)
def include_fingerprints(self, path):
self.fingerprints_to_include = []
@@ -40,11 +38,12 @@ class Filtering(object):
def filter_tor_circuits(self, analysis):
if self.fingerprints_to_include is None and
self.fingerprints_to_exclude is None:
return
-self.filters["tor/circuits"] = []
+filters = analysis.json_db.setdefault("filters", {})
+tor_circuits_filters = filters.setdefault("tor/circuits", [])
if self.fingerprints_to_include:
- self.filters["tor/circuits"].append({"name":
"include_fingerprints", "filepath": self.fingerprints_to_include_path })
+ tor_circuits_filters.append({"name": "include_fingerprints",
"filepath": self.fingerprints_to_include_path })
if self.fingerprints_to_exclude:
- self.filters["tor/circuits"].append({"name":
"exclude_fingerprints", "filepath": self.fingerprints_to_exclude_path })
+ tor_circuits_filters.append({"name": "exclude_fingerprints",
"filepath": self.fingerprints_to_exclude_path })
for source in analysis.get_nodes():
tor_circuits = analysis.get_tor_circuits(source)
filtered_circuit_ids = []
@@ -68,10 +67,9 @@ class Filtering(object):
tor_circuits[circuit_id] =
dict(sorted(tor_circuit.items()))
def apply_filters(self, input_path, output_dir, output_file):
-self.analysis = OPAnalysis.load(filename=input_path)
-self.filter_tor_circuits(self.analysis)
-self.analysis.json_db["filters"] = self.filters
-self.analysis.json_db["version"] = '4.0'
-self.analysis.json_db = dict(sorted(self.analysis.json_db.items()))
-self.analysis.save(filename=output_file, output_prefix=output_dir,
sort_keys=False)
+analysis = OPAnalysis.load(filename=input_path)
+self.filter_tor_circuits(analysis)
+analysis.json_db["version"] = '4.0'
+analysis.json_db = dict(sorted(analysis.json_db.items()))
+analysis.save(filename=output_file, output_prefix=output_dir,
sort_keys=False)
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/develop] Bump analysis version to 4.0
commit d0660c1847b46863783b7006efdac4df8103f123
Author: Ana Custura
Date: Thu Sep 10 12:54:41 2020 +0100
Bump analysis version to 4.0
---
onionperf/analysis.py | 4 ++--
onionperf/filtering.py | 1 +
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/onionperf/analysis.py b/onionperf/analysis.py
index 49c109c..907d451 100644
--- a/onionperf/analysis.py
+++ b/onionperf/analysis.py
@@ -24,7 +24,7 @@ class OPAnalysis(Analysis):
def __init__(self, nickname=None, ip_address=None):
super().__init__(nickname, ip_address)
-self.json_db = {'type': 'onionperf', 'version': '3.0', 'data': {}}
+self.json_db = {'type': 'onionperf', 'version': '4.0', 'data': {}}
self.torctl_filepaths = []
def add_torctl_file(self, filepath):
@@ -133,7 +133,7 @@ class OPAnalysis(Analysis):
if 'type' not in db or 'version' not in db:
logging.warning("'type' or 'version' not present in database")
return None
-elif db['type'] != 'onionperf' or str(db['version']) >= '4.':
+elif db['type'] != 'onionperf' or str(db['version']) >= '5.':
logging.warning("type or version not supported (type={0},
version={1})".format(db['type'], db['version']))
return None
else:
diff --git a/onionperf/filtering.py b/onionperf/filtering.py
index 0ba9ebc..e8ea164 100644
--- a/onionperf/filtering.py
+++ b/onionperf/filtering.py
@@ -71,6 +71,7 @@ class Filtering(object):
self.analysis = OPAnalysis.load(filename=input_path)
self.filter_tor_circuits(self.analysis)
self.analysis.json_db["filters"] = self.filters
+self.analysis.json_db["version"] = '4.0'
self.analysis.json_db = dict(sorted(self.analysis.json_db.items()))
self.analysis.save(filename=output_file, output_prefix=output_dir,
sort_keys=False)
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/develop] Fix matching of tor_circuit ids
commit 26ff79a3e57f1dfb0c961cceb167534bb39d17f3 Author: Ana Custura Date: Fri Aug 21 02:14:32 2020 +0100 Fix matching of tor_circuit ids --- onionperf/filtering.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onionperf/filtering.py b/onionperf/filtering.py index 435a1bc..204b161 100644 --- a/onionperf/filtering.py +++ b/onionperf/filtering.py @@ -73,7 +73,7 @@ class Filtering(object): for tor_stream in tor_streams_by_source_port[source_port]: if abs(unix_ts_end - tor_stream["unix_ts_end"]) < 150.0: circuit_id = tor_stream["circuit_id"] -if circuit_id and circuit_id in tor_circuits: +if circuit_id and str(circuit_id) in tor_circuits: tor_circuit = tor_circuits[circuit_id] path = tor_circuit["path"] keep = True ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/develop] Add path validation to filter interface
commit 1a81b3dff756cf197d661f931485443d1454c7c4
Author: Ana Custura
Date: Fri Aug 21 01:38:01 2020 +0100
Add path validation to filter interface
---
onionperf/onionperf | 3 +++
1 file changed, 3 insertions(+)
diff --git a/onionperf/onionperf b/onionperf/onionperf
index b1e7bd3..96c6869 100755
--- a/onionperf/onionperf
+++ b/onionperf/onionperf
@@ -439,6 +439,9 @@ def analyze(args):
def filter(args):
from onionperf.filtering import Filtering
+p = os.path.abspath(os.path.expanduser(args.input))
+if not os.path.exists(p):
+raise argparse.ArgumentTypeError("path '%s' does not exist" %
args.input)
filtering = Filtering()
filtering.read_input(args.input)
if args.include_fingerprints is not None:
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/develop] Fix naming of exclusion list
commit d9f8b8fe36fabb30bf96802130a68b9020f6a875 Author: Ana Custura Date: Fri Aug 21 03:07:30 2020 +0100 Fix naming of exclusion list --- onionperf/filtering.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onionperf/filtering.py b/onionperf/filtering.py index 204b161..ab96f1e 100644 --- a/onionperf/filtering.py +++ b/onionperf/filtering.py @@ -28,13 +28,13 @@ class Filtering(object): self.fingerprints_to_include.append(fingerprint) def exclude_fingerprints(self, path): -self.exclude_fingerprints = [] +self.fingerprints_to_exclude = [] with open(path, 'rt') as f: for line in f: fingerprint_match = self.fingerprint_pattern.match(line) if fingerprint_match: fingerprint = fingerprint_match.group(1).upper() -self.exclude_fingerprints.append(fingerprint) +self.fingerprints_to_exclude.append(fingerprint) def apply_filters(self): if self.fingerprints_to_include is None and self.fingerprints_to_exclude is None: ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/develop] Clarify filter terminology
commit a466221c5b947bb894a36fd3e5472397f1b0b03e Author: Ana Custura Date: Thu Sep 10 12:41:46 2020 +0100 Clarify filter terminology --- onionperf/filtering.py | 2 +- onionperf/visualization.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/onionperf/filtering.py b/onionperf/filtering.py index 7ef6168..0ba9ebc 100644 --- a/onionperf/filtering.py +++ b/onionperf/filtering.py @@ -64,7 +64,7 @@ class Filtering(object): keep = False break if not keep: -tor_circuits[circuit_id]["filtered"] = True +tor_circuits[circuit_id]["filtered_out"] = True tor_circuits[circuit_id] = dict(sorted(tor_circuit.items())) def apply_filters(self, input_path, output_dir, output_file): diff --git a/onionperf/visualization.py b/onionperf/visualization.py index f5bc03f..2cd3161 100644 --- a/onionperf/visualization.py +++ b/onionperf/visualization.py @@ -147,7 +147,7 @@ class TGenVisualization(Visualization): stream["error_code"] = "/".join(error_code_parts) if "filters" in analysis.json_db.keys() and analysis.json_db["filters"]["tor/circuits"]: - if tor_circuit and "filtered" not in tor_circuit.keys(): + if tor_circuit and "filtered_out" not in tor_circuit.keys(): streams.append(stream) else: streams.append(stream) ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/develop] Only try to load analysis files in an input directory
commit d21d41e4d504b48ac5617530830da320a89a9eed Author: Ana Custura Date: Sat Aug 29 15:10:32 2020 +0100 Only try to load analysis files in an input directory --- onionperf/onionperf | 11 ++- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/onionperf/onionperf b/onionperf/onionperf index 7c16aea..e3f49c8 100755 --- a/onionperf/onionperf +++ b/onionperf/onionperf @@ -458,11 +458,12 @@ def filter(args): output_dir, output_file = os.path.split(output_path) filtering.apply_filters(input_path=input_path, output_dir=output_dir, output_file=output_file) else: -for dirpath, dirnames, filenames in os.walk(input_path): -for filename in filenames: -input_file = os.path.join(dirpath, filename) -output_dir = os.path.join(output_path, os.path.relpath(dirpath, input_path)) -filtering.apply_filters(input_path=input_file, output_dir=output_dir, output_file=filename) +from onionperf import reprocessing +analyses = reprocessing.collect_logs(input_path, '*onionperf.analysis.*') +for analysis in analyses: +full_output_path = os.path.join(output_path, os.path.relpath(analysis, input_path)) +output_dir, output_file = os.path.split(full_output_path) +filtering.apply_filters(input_path=analysis, output_dir=output_dir, output_file=output_file) def visualize(args): from onionperf.visualization import TGenVisualization ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/develop] Add filter section to README.md
commit 52506522bc6dc679318e2af801f7c3a94b6947d0 Author: Ana Custura Date: Sat Aug 29 17:01:13 2020 +0100 Add filter section to README.md --- README.md | 23 +++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index ad53a9e..224972c 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ + [Troubleshooting](#troubleshooting) * [Analysis](#analysis) + [Analyzing measurement results](#analyzing-measurement-results) ++ [Filtering measurement results](#filtering-measurement-results) + [Visualizing measurement results](#visualizing-measurement-results) + [Interpreting the PDF output format](#interpreting-the-pdf-output-format) + [Interpreting the CSV output format](#interpreting-the-csv-output-format) @@ -251,6 +252,28 @@ OnionPerf's `analyze` mode has several command-line parameters for customizing t ```shell onionperf analyze --help ``` +### Filtering measurement results + +OnionPerf measurement results can be filtered based on Tor relay fingerprints. +The `filter` mode takes a list of fingerprints and one or more existing analysis files as inputs, and outputs new analysis files containing only the `tgen` results obtained over a Tor circuit path which includes or excludes fingerprints in the input list. + +Where excluding fingerprints, if ANY relay fingerprint is matched, the measurement is discarded. Where including fingerprints, ALL relay fingerprints in a path must match for the measurement to be retained. + +For example, the analysis file produced above can be filtered with the following command, which retains measurements based on fingerprints contained in the file 'fingerprints.txt': + +```shell +onionperf filter -i onionperf.analysis.json.xz -o filtered.onionperf.analysis.json.xz --include-fingerprints fingerprints.txt +``` + +The output analysis file is written to the path specified with `-o`. If processing a directory of analysis files, its structure and filenames are preserved under the path specified with '-o'. +Note that while the subcommand filters `tgen` measurements, it leaves `tgen` and `tor` summaries in the original analysis file unchanged. + +OnionPerf's `filter` command usage can be inspected with: + +```shell +onionperf filter --help +``` + ### Visualizing measurement results ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/develop] Prevent incorrectly keeping transfers for empty incl./excl. lists
commit efe2ef11eb91e803e44229f5ede0209ead0aae21 Author: Ana Custura Date: Sat Aug 29 14:18:01 2020 +0100 Prevent incorrectly keeping transfers for empty incl./excl. lists --- onionperf/filtering.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onionperf/filtering.py b/onionperf/filtering.py index b431eb9..9e7b34f 100644 --- a/onionperf/filtering.py +++ b/onionperf/filtering.py @@ -79,10 +79,10 @@ class Filtering(object): fingerprint_match = self.fingerprint_pattern.match(long_name) if fingerprint_match: fingerprint = fingerprint_match.group(1).upper() -if self.fingerprints_to_include and fingerprint not in self.fingerprints_to_include: +if self.fingerprints_to_include is not None and fingerprint not in self.fingerprints_to_include: keep = False break -if self.fingerprints_to_exclude and fingerprint in self.fingerprints_to_exclude: +if self.fingerprints_to_exclude is not None and fingerprint in self.fingerprints_to_exclude: keep = False break if keep: ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/develop] Also accept a directory in `onionperf filter -i`.
commit 053da92c69d4b3628f33e1dd610a1bc8f601cbe0
Author: Karsten Loesing
Date: Thu Aug 27 09:59:59 2020 +0200
Also accept a directory in `onionperf filter -i`.
And clarify that we're leaving statistics unchanged as part of the
filtering.
---
CHANGELOG.md | 4 ++--
onionperf/filtering.py | 10 +++---
onionperf/onionperf| 35 ---
3 files changed, 29 insertions(+), 20 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c57695e..ebd43e1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,8 +1,8 @@
# Changes in version 0.7 - 2020-??-??
- Add a new `onionperf filter` mode that takes an OnionPerf analysis
- results file as input, applies filters, and produces a new
- OnionPerf analysis results file as output.
+ results file or directory as input, applies filters, and produces
+ new OnionPerf analysis results file(s) as output.
# Changes in version 0.6 - 2020-08-08
diff --git a/onionperf/filtering.py b/onionperf/filtering.py
index ab96f1e..b431eb9 100644
--- a/onionperf/filtering.py
+++ b/onionperf/filtering.py
@@ -15,9 +15,6 @@ class Filtering(object):
self.fingerprints_to_exclude = None
self.fingerprint_pattern = re.compile("\$?([0-9a-fA-F]{40})")
-def read_input(self, path):
-self.analysis = OPAnalysis.load(filename=path)
-
def include_fingerprints(self, path):
self.fingerprints_to_include = []
with open(path, 'rt') as f:
@@ -36,7 +33,8 @@ class Filtering(object):
fingerprint = fingerprint_match.group(1).upper()
self.fingerprints_to_exclude.append(fingerprint)
-def apply_filters(self):
+def apply_filters(self, input_path, output_dir, output_file):
+self.analysis = OPAnalysis.load(filename=input_path)
if self.fingerprints_to_include is None and
self.fingerprints_to_exclude is None:
return
for source in self.analysis.get_nodes():
@@ -94,7 +92,5 @@ class Filtering(object):
retained_tgen_transfers[transfer_id] = transfer_data
self.analysis.set_tgen_streams(source, retained_tgen_streams)
self.analysis.set_tgen_transfers(source, retained_tgen_transfers)
-
-def write_output(self, path):
-self.analysis.save(filename=path)
+self.analysis.save(filename=output_file, output_prefix=output_dir)
diff --git a/onionperf/onionperf b/onionperf/onionperf
index 96c6869..7c16aea 100755
--- a/onionperf/onionperf
+++ b/onionperf/onionperf
@@ -76,8 +76,11 @@ Analyze Tor and TGen output
"""
DESC_FILTER = """
-Takes an OnionPerf analysis results file as input, applies filters,
-and produces a new OnionPerf analysis results file as output.
+Takes an OnionPerf analysis results file or directory as input, applies
filters,
+and produces new OnionPerf analysis results file(s) as output.
+
+This subcommand only filters measurements in `data/[source]/tgen/transfers`
+and `data/[source]/tgen/streams`, but leaves any summaries unchanged.
"""
HELP_FILTER = """
Filter OnionPerf analysis results
@@ -295,7 +298,8 @@ files generated by this script will be written""",
filter_parser.set_defaults(func=filter, formatter_class=my_formatter_class)
filter_parser.add_argument('-i', '--input',
-help="""read the OnionPerf analysis results at PATH as input""",
+help="""a file or directory PATH from which OnionPerf analysis results
+files are read""",
metavar="PATH", required="True",
action="store", dest="input")
@@ -314,8 +318,8 @@ files generated by this script will be written""",
default=None)
filter_parser.add_argument('-o', '--output',
-help="""write the filtered output OnionPerf analysis results file to
-PATH""",
+help="""a file or directory PATH where filtered output OnionPerf
+analysis results files are written""",
metavar="PATH", required="True",
action="store", dest="output")
@@ -439,17 +443,26 @@ def analyze(args):
def filter(args):
from onionperf.filtering import Filtering
-p = os.path.abspath(os.path.expanduser(args.input))
-if not os.path.exists(p):
-raise argparse.ArgumentTypeError("path '%s' does not exist" %
args.input)
+input_path = os.path.abspath(os.path.expanduser(args.input))
+if not os.path.exists(input_path):
+raise argparse.ArgumentTypeError("input path '%s' does not exist" %
args.input)
+output_path = os.path.abspath(os.path.expanduser(args.output))
+if os.path.exists(output_path)
[tor-commits] [onionperf/master] Bump version to 0.7.
commit c8275b25e4afda9328634ec6be56ff46c7ee1cfe
Author: Karsten Loesing
Date: Tue Sep 1 22:30:12 2020 +0200
Bump version to 0.7.
---
CHANGELOG.md | 2 +-
setup.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e0253c5..80fb14d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,4 @@
-# Changes in version 0.7 - 2020-??-??
+# Changes in version 0.7 - 2020-09-01
- Add `onionperf measure --drop-guards` parameter to use and drop
guards and circuit build timeouts every given number of hours, if
diff --git a/setup.py b/setup.py
index 7fd7f85..d72b27a 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@ with open('requirements.txt') as f:
install_requires = f.readlines()
setup(name='OnionPerf',
- version='0.6',
+ version='0.7',
description='A utility to monitor, measure, analyze, and visualize the
performance of Tor and Onion Services',
author='Rob Jansen',
url='https://github.com/robgjansen/onionperf/',
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/master] Let TGen client finish by itself in one-shot mode.
commit 959cf3689106189001a83c7e58dc40e10497a081
Author: Philipp Winter
Date: Fri Aug 7 14:48:58 2020 -0700
Let TGen client finish by itself in one-shot mode.
We tell TGen client to finish on its own by passing the count option to
the end node:
https://github.com/shadow/tgen/blob/master/doc/TGen-Options.md#end-options
This patch adds another argument to the function watchdog_thread_task(),
no_relaunch, which instructs the function to not re-launch its process
if it fails.
---
onionperf/measurement.py | 45 +++--
onionperf/model.py | 3 ++-
2 files changed, 25 insertions(+), 23 deletions(-)
diff --git a/onionperf/measurement.py b/onionperf/measurement.py
index e2d8d1c..d699292 100644
--- a/onionperf/measurement.py
+++ b/onionperf/measurement.py
@@ -50,10 +50,11 @@ def readline_thread_task(instream, q):
# wait for lines from stdout until the EOF
for line in iter(instream.readline, b''): q.put(line)
-def watchdog_thread_task(cmd, cwd, writable, done_ev, send_stdin,
ready_search_str, ready_ev):
+def watchdog_thread_task(cmd, cwd, writable, done_ev, send_stdin,
ready_search_str, ready_ev, no_relaunch):
-# launch or re-launch our sub process until we are told to stop
-# if we fail too many times in too short of time, give up and exit
+# launch or re-launch (or don't re-launch, if no_relaunch is set) our sub
+# process until we are told to stop if we fail too many times in too short
+# of time, give up and exit
failure_times = []
pause_time_seconds = 0
while done_ev.is_set() is False:
@@ -105,6 +106,10 @@ def watchdog_thread_task(cmd, cwd, writable, done_ev,
send_stdin, ready_search_s
subp.wait()
elif done_ev.is_set():
logging.info("command '{}' finished as expected".format(cmd))
+elif no_relaunch:
+logging.info("command '{}' finished on its own".format(cmd))
+# our command finished on its own. time to terminate.
+done_ev.set()
else:
logging.warning("command '{}' finished before
expected".format(cmd))
now = time.time()
@@ -284,15 +289,9 @@ class Measurement(object):
time.sleep(1)
while True:
if tgen_model.num_transfers:
-downloads = 0
-while True:
-downloads =
self.__get_download_count(tgen_client_writable.filename)
-time.sleep(1)
-if downloads >= tgen_model.num_transfers:
-logging.info("Onionperf has downloaded %d
files and will now shut down." % tgen_model.num_transfers)
-break
-else:
-continue
+# This function blocks until our TGen client process
+# terminated on its own.
+self.__wait_for_tgen_client()
break
if self.__is_alive():
@@ -366,7 +365,10 @@ class Measurement(object):
logging.info("Logging TGen {1} process output to
{0}".format(tgen_logpath, name))
tgen_cmd = "{0} {1}".format(self.tgen_bin_path, tgen_confpath)
-tgen_args = (tgen_cmd, tgen_datadir, tgen_writable, self.done_event,
None, None, None)
+# If we're running in "one-shot mode", TGen client will terminate on
+# its own and we don't need our watchdog to restart the process.
+no_relaunch = (name == "client" and tgen_model_conf.num_transfers)
+tgen_args = (tgen_cmd, tgen_datadir, tgen_writable, self.done_event,
None, None, None, no_relaunch)
tgen_watchdog = threading.Thread(target=watchdog_thread_task,
name="tgen_{0}_watchdog".format(name), args=tgen_args)
tgen_watchdog.start()
self.threads.append(tgen_watchdog)
@@ -464,7 +466,7 @@ WarnUnsafeSocks 0\nSafeLogging 0\nMaxCircuitDirtiness 60
seconds\nDataDirectory
tor_stdin_bytes = str_tools._to_bytes(tor_config)
tor_ready_str = "Bootstrapped 100"
tor_ready_ev = threading.Event()
-tor_args = (tor_cmd, tor_datadir, tor_writable, self.done_event,
tor_stdin_bytes, tor_ready_str, tor_ready_ev)
+tor_args = (tor_cmd, tor_datadir, tor_writable, self.done_event,
tor_stdin_bytes, tor_ready_str, tor_ready_ev, False)
tor_watchdog = threading.Thread(target=watchdog_thread_task,
name="tor_{0}_watchdog".format(name), args=tor_args)
tor_watchdog.start()
self.threads.append(tor_watchdog)
@@ -491,14 +493,13 @@ WarnUnsafeSocks 0\nSafeLogging 0\nMaxCircuitDirtiness 60
seconds\nDataDirectory
return tor_writable, torctl_writable
-def __get_download_count(self, tgen_logpath):
-count = 0
-if tgen_logpath is not None and os.path.exists(tgen_logpath):
-
[tor-commits] [onionperf/master] Make some tweaks to new TGen model.
commit b8f1e5c2695c097a7494f7975403664c0c833825
Author: Karsten Loesing
Date: Sun Aug 16 22:03:34 2020 +0200
Make some tweaks to new TGen model.
- Change timeout back to 270 seconds and stallout back to 0 seconds.
- Change initial pause to 300 seconds to keep default behavior
unchanged.
- Change model, so that pause_between starts in parallel to a stream,
not when the stream is completed. This is the same behavior as
before.
Also add a change log entry for all changes.
---
CHANGELOG.md| 7 +++
onionperf/model.py | 20
onionperf/onionperf | 2 +-
3 files changed, 16 insertions(+), 13 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0c4c4f2..ac6897b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,10 @@
+# Changes in version 0.7 - 2020-??-??
+
+ - Remove the `onionperf measure --oneshot` switch and replace it with
+ new switches `--tgen-pause-initial`, `--tgen-pause-between`,
+ `--tgen-transfer-size`, and `--tgen-num-transfers ` to further
+ configure the generated TGen model.
+
# Changes in version 0.6 - 2020-??-??
- Update to TGen 1.0.0, use TGenTools for parsing TGen log files, and
diff --git a/onionperf/model.py b/onionperf/model.py
index d45763e..fde587f 100644
--- a/onionperf/model.py
+++ b/onionperf/model.py
@@ -43,8 +43,8 @@ class TGenLoadableModel(TGenModel):
class TGenModelConf(object):
"""Represents a TGen traffic model configuration."""
-def __init__(self, pause_initial=0, num_transfers=1, transfer_size="5 MiB",
- continuous_transfers=False, pause_between=5, port=None,
servers=[],
+def __init__(self, pause_initial=300, num_transfers=1, transfer_size="5
MiB",
+ continuous_transfers=False, pause_between=300, port=None,
servers=[],
socks_port=None):
self.pause_initial = pause_initial
self.pause_between = pause_between
@@ -103,28 +103,24 @@ class TorperfModel(GeneratableTGenModel):
g.add_node("stream",
sendsize="0",
recvsize=self.config.transfer_size,
- timeout="15 seconds",
- stallout="10 seconds")
+ timeout="270 seconds",
+ stallout="0 seconds")
g.add_node("pause_between",
time="%d seconds" % self.config.pause_between)
g.add_edge("start", "pause_initial")
g.add_edge("pause_initial", "stream")
+g.add_edge("pause_initial", "pause_between")
+g.add_edge("pause_between", "stream")
+g.add_edge("pause_between", "pause_between")
# only add an end node if we need to stop
-if self.config.continuous_transfers:
-# continuous mode, i.e., no end node
-g.add_edge("stream", "pause_between")
-else:
+if not self.config.continuous_transfers:
# one-shot mode, i.e., end after configured number of transfers
g.add_node("end",
count="%d" % self.config.num_transfers)
# check for end condition after every transfer
g.add_edge("stream", "end")
-# if end condition not met, pause
-g.add_edge("end", "pause_between")
-
-g.add_edge("pause_between", "stream")
return g
diff --git a/onionperf/onionperf b/onionperf/onionperf
index a49982b..6a16da2 100755
--- a/onionperf/onionperf
+++ b/onionperf/onionperf
@@ -194,7 +194,7 @@ def main():
help="""the number of seconds TGen should wait before walking through
its action graph""",
metavar="N", type=int,
action="store", dest="tgenpauseinitial",
-default=5)
+default=300)
measure_parser.add_argument('--tgen-pause-between',
help="""the number of seconds TGen should wait in between two
transfers""",
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/master] Merge remote-tracking branch 'karsten/task-33399-2' into develop
commit 8fe84702dea4b2e4640c0817e505e6c449e7b9a8 Merge: f496ede 9d76ca4 Author: Karsten Loesing Date: Tue Sep 1 22:28:52 2020 +0200 Merge remote-tracking branch 'karsten/task-33399-2' into develop CHANGELOG.md | 3 ++- onionperf/monitor.py | 12 +++- onionperf/onionperf | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/master] Tweak #33399 patch.
commit dfec0b8960ac214f63985dfc317ffce750ef5922 Author: Karsten Loesing Date: Thu Aug 20 15:40:25 2020 +0200 Tweak #33399 patch. - Add a change log entry. - Pick are more sensible default for `drop_guards_interval_hours`, also to fix unit tests. --- CHANGELOG.md | 5 + onionperf/measurement.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8a86ee..c31a40e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +# Changes in version 0.7 - 2020-??-?? + + - Add `onionperf measure --drop-guards` parameter to use and drop + guards after a given number of hours. Implements #33399. + # Changes in version 0.6 - 2020-08-08 - Update to TGen 1.0.0, use TGenTools for parsing TGen log files, and diff --git a/onionperf/measurement.py b/onionperf/measurement.py index 1a5f3bb..709fbc6 100644 --- a/onionperf/measurement.py +++ b/onionperf/measurement.py @@ -173,7 +173,7 @@ def logrotate_thread_task(writables, tgen_writable, torctl_writable, docroot, ni class Measurement(object): -def __init__(self, tor_bin_path, tgen_bin_path, datadir_path, privatedir_path, nickname, oneshot, additional_client_conf=None, torclient_conf_file=None, torserver_conf_file=None, single_onion=False, drop_guards_interval_hours=None): +def __init__(self, tor_bin_path, tgen_bin_path, datadir_path, privatedir_path, nickname, oneshot, additional_client_conf=None, torclient_conf_file=None, torserver_conf_file=None, single_onion=False, drop_guards_interval_hours=0): self.tor_bin_path = tor_bin_path self.tgen_bin_path = tgen_bin_path self.datadir_path = datadir_path ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/master] Tweak --drop-guards switch.
commit 9d76ca4b3847a10c9565d12cf130933a1076297b
Author: Karsten Loesing
Date: Tue Sep 1 17:02:47 2020 +0200
Tweak --drop-guards switch.
This commit tweaks the recently added --drop-guards switch as follows:
- Guards are dropped right at startup and then every N hours.
Otherwise we might not receive the first round of GUARD NEW/UP
events. It's unclear why we don't receive those events, but finding
out might be time-consuming whereas dropping guards at startup is
basically free.
- Right after guards are dropped, circuit build timeouts are dropped,
too, if supported by the Tor version. If the Tor version does not
support this, there's going to be a warning, and the control log
will simply not contain BUILDTIMEOUT_SET events.
Still part of the reopened tpo/metrics/onionperf#33399.
---
CHANGELOG.md | 3 ++-
onionperf/monitor.py | 12 +++-
onionperf/onionperf | 2 +-
3 files changed, 10 insertions(+), 7 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a212842..e0253c5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,7 +1,8 @@
# Changes in version 0.7 - 2020-??-??
- Add `onionperf measure --drop-guards` parameter to use and drop
- guards after a given number of hours. Implements #33399.
+ guards and circuit build timeouts every given number of hours, if
+ supported by the Tor version. Implements #33399.
- Remove the `onionperf measure --oneshot` switch and replace it with
new switches `--tgen-pause-initial`, `--tgen-pause-between`,
`--tgen-transfer-size`, and `--tgen-num-transfers ` to further
diff --git a/onionperf/monitor.py b/onionperf/monitor.py
index b5e6390..886a9a0 100644
--- a/onionperf/monitor.py
+++ b/onionperf/monitor.py
@@ -57,21 +57,23 @@ class TorMonitor(object):
interval_count = 0
if newnym_interval_seconds is not None:
next_newnym = newnym_interval_seconds
-if drop_guards_interval_hours > 0:
-next_drop_guards = drop_guards_interval_hours * 3600
+next_drop_guards = 0
while done_ev is None or not done_ev.is_set():
# if self.filepath != '-' and
os.path.exists(self.filepath):
#with open(self.filepath, 'rb') as sizef:
#msg = "tor-ctl-logger[port={0}] logged {1} bytes
to {2}, press CTRL-C to quit".format(self.tor_ctl_port,
os.fstat(sizef.fileno()).st_size, self.filepath)
#logging.info(msg)
+if drop_guards_interval_hours > 0 and interval_count >=
next_drop_guards:
+next_drop_guards += drop_guards_interval_hours * 3600
+torctl.drop_guards()
+drop_timeouts_response = torctl.msg("DROPTIMEOUTS")
+if not drop_timeouts_response.is_ok():
+self.__log(self.writable, "[WARNING] unrecognized
command DROPTIMEOUTS in tor\n")
sleep(1)
interval_count += 1
if newnym_interval_seconds is not None and interval_count
>= next_newnym:
next_newnym += newnym_interval_seconds
torctl.signal(Signal.NEWNYM)
-if drop_guards_interval_hours > 0 and interval_count >=
next_drop_guards:
-next_drop_guards += drop_guards_interval_hours * 3600
-torctl.drop_guards()
except KeyboardInterrupt:
pass # the user hit ctrl+c
diff --git a/onionperf/onionperf b/onionperf/onionperf
index e6aa44a..d3d8e31 100755
--- a/onionperf/onionperf
+++ b/onionperf/onionperf
@@ -215,7 +215,7 @@ def main():
default=0)
measure_parser.add_argument('--drop-guards',
-help="""Use and drop guards every N > 0 hours, or do not use guards at
all if N = 0""",
+help="""Use and drop guards and circuit build timeouts every N > 0
hours, or do not use guards at all and never drop circuit build timeouts if N =
0""",
metavar="N", type=type_nonnegative_integer,
action="store", dest="drop_guards_interval_hours",
default=0)
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/master] Apply 1 suggestion(s) to 1 file(s)
commit a6aa4189e04ee1b05dbdb68b90f24e14bf3443ac
Author: Philipp Winter
Date: Fri Aug 14 16:47:04 2020 +
Apply 1 suggestion(s) to 1 file(s)
---
onionperf/model.py | 54 ++
1 file changed, 22 insertions(+), 32 deletions(-)
diff --git a/onionperf/model.py b/onionperf/model.py
index bdd5a53..b589249 100644
--- a/onionperf/model.py
+++ b/onionperf/model.py
@@ -104,40 +104,30 @@ class TorperfModel(GeneratableTGenModel):
# "One-shot mode," i.e., onionperf will stop after the given number of
# iterations. The idea is:
# start -> pause -> stream-1 -> pause-1 -> ... -> stream-n -> pause-n
-> end
-if self.config.num_transfers > 0:
-for i in range(self.config.num_transfers):
-g.add_node("stream-%d" % i,
- sendsize="0",
- recvsize=self.config.transfer_size,
- timeout="15 seconds",
- stallout="10 seconds")
-g.add_node("pause-%d" % i,
- time="%d seconds" %
self.config.inter_transfer_pause)
-
-g.add_edge("stream-%d" % i, "pause-%d" % i)
-if i > 0:
-g.add_edge("pause-%d" % (i-1), "stream-%d" % i)
-
+g.add_node("stream",
+ sendsize="0",
+ recvsize=self.config.transfer_size,
+ timeout="15 seconds",
+ stallout="10 seconds")
+g.add_node("pause_between",
+ time="%d seconds" % self.config.inter_transfer_pause)
+
+g.add_edge("pause_initial", "stream")
+
+# only add an end node if we need to stop
+if self.config.continuous_transfers:
+# continuous mode, i.e., no end node
+g.add_edge("stream", "pause_between")
+else:
+# one-shot mode, i.e., end after configured number of transfers
g.add_node("end",
count=str(self.config.num_transfers))
-g.add_edge("pause", "stream-0")
-g.add_edge("pause-%d" % (self.config.num_transfers - 1), "end")
-
-# Continuous mode, i.e., onionperf will not stop. The idea is:
-# start -> pause -> stream -> pause
-# ^ |
-# +---+
-elif self.config.continuous_transfers:
-g.add_node("stream",
- sendsize="0",
- recvsize=self.config.transfer_size,
- timeout="15 seconds",
- stallout="10 seconds")
-g.add_node("pause",
- time="%d seconds" % self.config.inter_transfer_pause)
-g.add_edge("pause", "stream")
-g.add_edge("stream", "pause")
-g.add_edge("pause", "stream")
+# check for end condition after every transfer
+g.add_edge("stream", "end")
+# if end condition not met, pause
+g.add_edge("end", "pause_between")
+
+g.add_edge("pause_between", "stream")
return g
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/master] Measure static guard nodes.
commit e9fd47d95db102b3a7ace36fa412e18d182c5fa4
Author: Karsten Loesing
Date: Tue Jun 16 21:30:07 2020 +0200
Measure static guard nodes.
Add --drop-guards parameter to use and drop guards after a given
number of hours.
Implements #33399.
---
onionperf/measurement.py | 7 ---
onionperf/monitor.py | 18 +-
onionperf/onionperf | 9 -
3 files changed, 25 insertions(+), 9 deletions(-)
diff --git a/onionperf/measurement.py b/onionperf/measurement.py
index 4a58bc4..899b277 100644
--- a/onionperf/measurement.py
+++ b/onionperf/measurement.py
@@ -172,7 +172,7 @@ def logrotate_thread_task(writables, tgen_writable,
torctl_writable, docroot, ni
class Measurement(object):
-def __init__(self, tor_bin_path, tgen_bin_path, datadir_path,
privatedir_path, nickname, oneshot, additional_client_conf=None,
torclient_conf_file=None, torserver_conf_file=None, single_onion=False):
+def __init__(self, tor_bin_path, tgen_bin_path, datadir_path,
privatedir_path, nickname, oneshot, additional_client_conf=None,
torclient_conf_file=None, torserver_conf_file=None, single_onion=False,
drop_guards_interval_hours=None):
self.tor_bin_path = tor_bin_path
self.tgen_bin_path = tgen_bin_path
self.datadir_path = datadir_path
@@ -188,6 +188,7 @@ class Measurement(object):
self.torclient_conf_file = torclient_conf_file
self.torserver_conf_file = torserver_conf_file
self.single_onion = single_onion
+self.drop_guards_interval_hours = drop_guards_interval_hours
def run(self, do_onion=True, do_inet=True, client_tgen_listen_port=5,
client_tgen_connect_ip='0.0.0.0', client_tgen_connect_port=8080,
client_tor_ctl_port=59050, client_tor_socks_port=59000,
server_tgen_listen_port=8080, server_tor_ctl_port=59051,
server_tor_socks_port=59001):
@@ -388,7 +389,7 @@ WarnUnsafeSocks 0\nSafeLogging 0\nMaxCircuitDirtiness 60
seconds\nDataDirectory
tor_config = tor_config + f.read()
if name == "client" and self.additional_client_conf:
tor_config += self.additional_client_conf
-if not 'UseEntryGuards' in tor_config and not 'UseBridges' in
tor_config:
+if not 'UseEntryGuards' in tor_config and not 'UseBridges' in
tor_config and self.drop_guards_interval_hours == 0:
tor_config += "UseEntryGuards 0\n"
if name == "server" and self.single_onion:
tor_config += "HiddenServiceSingleHopMode
1\nHiddenServiceNonAnonymousMode 1\n"
@@ -467,7 +468,7 @@ WarnUnsafeSocks 0\nSafeLogging 0\nMaxCircuitDirtiness 60
seconds\nDataDirectory
torctl_events = [e for e in monitor.get_supported_torctl_events() if e
not in ['DEBUG', 'INFO', 'NOTICE', 'WARN', 'ERR']]
newnym_interval_seconds = 300
-torctl_args = (control_port, torctl_writable, torctl_events,
newnym_interval_seconds, self.done_event)
+torctl_args = (control_port, torctl_writable, torctl_events,
newnym_interval_seconds, self.drop_guards_interval_hours, self.done_event)
torctl_helper = threading.Thread(target=monitor.tor_monitor_run,
name="torctl_{0}_helper".format(name), args=torctl_args)
torctl_helper.start()
self.threads.append(torctl_helper)
diff --git a/onionperf/monitor.py b/onionperf/monitor.py
index 5387bff..ac6fea9 100644
--- a/onionperf/monitor.py
+++ b/onionperf/monitor.py
@@ -22,7 +22,7 @@ class TorMonitor(object):
self.writable = writable
self.events = events
-def run(self, newnym_interval_seconds=None, done_ev=None):
+def run(self, newnym_interval_seconds=None, drop_guards_interval_hours=0,
done_ev=None):
with Controller.from_port(port=self.tor_ctl_port) as torctl:
torctl.authenticate()
@@ -54,6 +54,10 @@ class TorMonitor(object):
# let stem run its threads and log all of the events, until user
interrupts
try:
interval_count = 0
+if newnym_interval_seconds is not None:
+next_newnym = newnym_interval_seconds
+if drop_guards_interval_hours > 0:
+next_drop_guards = drop_guards_interval_hours * 3600
while done_ev is None or not done_ev.is_set():
# if self.filepath != '-' and
os.path.exists(self.filepath):
#with open(self.filepath, 'rb') as sizef:
@@ -61,9 +65,13 @@ class TorMonitor(object):
#logging.info(msg)
sleep(1)
interval_count += 1
-if newnym_interval_seconds is not None and interval_count
>= newnym_interval_seconds:
-interval_count = 0
+if newnym_interval_seconds is not None and interval_count
>= next_newnym:
+next_newnym += new
[tor-commits] [onionperf/master] Merge branch 'phw-enhancement-33432-3' into develop
commit c707674ba9b0d7931038c12bbe2d01585a88eb22
Merge: dfec0b8 b8f1e5c
Author: Karsten Loesing
Date: Thu Aug 27 10:38:21 2020 +0200
Merge branch 'phw-enhancement-33432-3' into develop
CHANGELOG.md| 4 ++
onionperf/measurement.py| 139 +++-
onionperf/model.py | 98 ++---
onionperf/onionperf | 60
onionperf/tests/test_measurement.py | 12 ++--
5 files changed, 185 insertions(+), 128 deletions(-)
diff --cc CHANGELOG.md
index c31a40e,ac6897b..4b7fcb2
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@@ -1,9 -1,11 +1,13 @@@
# Changes in version 0.7 - 2020-??-??
+ - Add `onionperf measure --drop-guards` parameter to use and drop
+ guards after a given number of hours. Implements #33399.
+ - Remove the `onionperf measure --oneshot` switch and replace it with
+new switches `--tgen-pause-initial`, `--tgen-pause-between`,
+`--tgen-transfer-size`, and `--tgen-num-transfers ` to further
+configure the generated TGen model.
-# Changes in version 0.6 - 2020-??-??
+# Changes in version 0.6 - 2020-08-08
- Update to TGen 1.0.0, use TGenTools for parsing TGen log files, and
update analysis results file version to 3.0. Implements #33974.
diff --cc onionperf/measurement.py
index 709fbc6,d699292..f198be0
--- a/onionperf/measurement.py
+++ b/onionperf/measurement.py
@@@ -173,7 -188,7 +188,7 @@@ def logrotate_thread_task(writables, tg
class Measurement(object):
- def __init__(self, tor_bin_path, tgen_bin_path, datadir_path,
privatedir_path, nickname, oneshot, additional_client_conf=None,
torclient_conf_file=None, torserver_conf_file=None, single_onion=False,
drop_guards_interval_hours=0):
-def __init__(self, tor_bin_path, tgen_bin_path, datadir_path,
privatedir_path, nickname, additional_client_conf=None,
torclient_conf_file=None, torserver_conf_file=None, single_onion=False):
++def __init__(self, tor_bin_path, tgen_bin_path, datadir_path,
privatedir_path, nickname, additional_client_conf=None,
torclient_conf_file=None, torserver_conf_file=None, single_onion=False,
drop_guards_interval_hours=0):
self.tor_bin_path = tor_bin_path
self.tgen_bin_path = tgen_bin_path
self.datadir_path = datadir_path
@@@ -189,13 -203,11 +203,12 @@@
self.torclient_conf_file = torclient_conf_file
self.torserver_conf_file = torserver_conf_file
self.single_onion = single_onion
+self.drop_guards_interval_hours = drop_guards_interval_hours
- def run(self, do_onion=True, do_inet=True, client_tgen_listen_port=5,
client_tgen_connect_ip='0.0.0.0', client_tgen_connect_port=8080,
client_tor_ctl_port=59050, client_tor_socks_port=59000,
- server_tgen_listen_port=8080, server_tor_ctl_port=59051,
server_tor_socks_port=59001):
+ def run(self, do_onion=True, do_inet=True, tgen_model=None,
tgen_client_conf=None, tgen_server_conf=None):
'''
- only `server_tgen_listen_port` are "public" and need to be opened on
the firewall.
- if `client_tgen_connect_port` != `server_tgen_listen_port`, then you
should have installed a forwarding rule in the firewall.
+ only `tgen_server_conf.listen_port` are "public" and need to be
opened on the firewall.
+ if `tgen_client_conf.connect_port` != `tgen_server_conf.listen_port`,
then you should have installed a forwarding rule in the firewall.
all ports need to be unique though, and unique among multiple
onionperf instances.
here are some sane defaults:
diff --cc onionperf/onionperf
index 641db70,6a16da2..e6aa44a
--- a/onionperf/onionperf
+++ b/onionperf/onionperf
@@@ -195,12 -190,30 +190,36 @@@ def main()
action="store", dest="tgenconnectport",
default=8080)
+ measure_parser.add_argument('--tgen-pause-initial',
+ help="""the number of seconds TGen should wait before walking through
its action graph""",
+ metavar="N", type=int,
+ action="store", dest="tgenpauseinitial",
+ default=300)
+
+ measure_parser.add_argument('--tgen-pause-between',
+ help="""the number of seconds TGen should wait in between two
transfers""",
+ metavar="N", type=int,
+ action="store", dest="tgenpausebetween",
+ default=300)
+
+ measure_parser.add_argument('--tgen-transfer-size',
+ help="""the size of the file transfer that TGen will perform (e.g.,
'5 MiB' or '10 KiB')""",
+ metavar="STRING", type=str,
+ action="store", dest="tgentransfersize",
+ default="5 MiB")
+
+ measure_parser.add_argument('--tgen-num-transfers',
[tor-commits] [onionperf/master] Rename variables for consistency and clarity.
commit 86f746c3c6ee6dc3be342f53fea90e6c02d39991
Author: Philipp Winter
Date: Fri Aug 14 10:49:09 2020 -0700
Rename variables for consistency and clarity.
initial_pause-> pause_initial
inter_transfer_pause -> pause_between
Thanks to Rob for the suggestion.
---
onionperf/model.py | 19 ---
onionperf/onionperf | 12 ++--
2 files changed, 14 insertions(+), 17 deletions(-)
diff --git a/onionperf/model.py b/onionperf/model.py
index b589249..3bfe35f 100644
--- a/onionperf/model.py
+++ b/onionperf/model.py
@@ -43,14 +43,14 @@ class TGenLoadableModel(TGenModel):
class TGenModelConf(object):
"""Represents a TGen traffic model configuration."""
-def __init__(self, initial_pause=0, num_transfers=1, transfer_size="5 MiB",
- continuous_transfers=False, inter_transfer_pause=5,
port=None, servers=[],
+def __init__(self, pause_initial=0, num_transfers=1, transfer_size="5 MiB",
+ continuous_transfers=False, pause_between=5, port=None,
servers=[],
socks_port=None):
-self.initial_pause = initial_pause
+self.pause_initial = pause_initial
+self.pause_between = pause_between
self.num_transfers = num_transfers
self.transfer_size = transfer_size
self.continuous_transfers = continuous_transfers
-self.inter_transfer_pause = inter_transfer_pause
self.port = port
self.servers = servers
self.socks_port = socks_port
@@ -98,20 +98,17 @@ class TorperfModel(GeneratableTGenModel):
loglevel="info",
heartbeat="1 minute")
-g.add_node("pause", time="%d seconds" % self.config.initial_pause)
-g.add_edge("start", "pause")
-
-# "One-shot mode," i.e., onionperf will stop after the given number of
-# iterations. The idea is:
-# start -> pause -> stream-1 -> pause-1 -> ... -> stream-n -> pause-n
-> end
+g.add_node("pause_initial",
+ time="%d seconds" % self.config.pause_initial)
g.add_node("stream",
sendsize="0",
recvsize=self.config.transfer_size,
timeout="15 seconds",
stallout="10 seconds")
g.add_node("pause_between",
- time="%d seconds" % self.config.inter_transfer_pause)
+ time="%d seconds" % self.config.pause_between)
+g.add_edge("start", "pause_initial")
g.add_edge("pause_initial", "stream")
# only add an end node if we need to stop
diff --git a/onionperf/onionperf b/onionperf/onionperf
index d95e691..a49982b 100755
--- a/onionperf/onionperf
+++ b/onionperf/onionperf
@@ -190,16 +190,16 @@ def main():
action="store", dest="tgenconnectport",
default=8080)
-measure_parser.add_argument('--tgen-start-pause',
+measure_parser.add_argument('--tgen-pause-initial',
help="""the number of seconds TGen should wait before walking through
its action graph""",
metavar="N", type=int,
-action="store", dest="tgenstartpause",
+action="store", dest="tgenpauseinitial",
default=5)
-measure_parser.add_argument('--tgen-intertransfer-pause',
+measure_parser.add_argument('--tgen-pause-between',
help="""the number of seconds TGen should wait in between two
transfers""",
metavar="N", type=int,
-action="store", dest="tgenintertransferpause",
+action="store", dest="tgenpausebetween",
default=300)
measure_parser.add_argument('--tgen-transfer-size',
@@ -377,11 +377,11 @@ def measure(args):
tor_ctl_port=server_tor_ctl_port,
tor_socks_port=server_tor_socks_port)
-tgen_model = TGenModelConf(initial_pause=args.tgenstartpause,
+tgen_model = TGenModelConf(pause_initial=args.tgenpauseinitial,
transfer_size=args.tgentransfersize,
num_transfers=args.tgennumtransfers,
continuous_transfers=args.tgennumtransfers
== 0,
-
inter_transfer_pause=args.tgenintertransferpause)
+ pause_between=args.tgenpausebetween)
meas = Measurement(args.torpath,
args.tgenpath,
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/master] Add issue number to change log.
commit f496edec6fec8adc0b4e5c0454989c2bc06312e3 Author: Karsten Loesing Date: Thu Aug 27 11:09:07 2020 +0200 Add issue number to change log. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b7fcb2..a212842 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ - Remove the `onionperf measure --oneshot` switch and replace it with new switches `--tgen-pause-initial`, `--tgen-pause-between`, `--tgen-transfer-size`, and `--tgen-num-transfers ` to further - configure the generated TGen model. + configure the generated TGen model. Implemets #33432. # Changes in version 0.6 - 2020-08-08 ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/master] Merge branch 'task-33399' into develop
commit 4ff257c4270c0d1e5fd0f1ef76e640696ec2c514 Merge: e333be2 e9fd47d Author: Karsten Loesing Date: Thu Aug 20 15:05:23 2020 +0200 Merge branch 'task-33399' into develop onionperf/measurement.py | 7 --- onionperf/monitor.py | 18 +- onionperf/onionperf | 9 - 3 files changed, 25 insertions(+), 9 deletions(-) ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/master] Use format string for consistency.
commit 1eea5e10700c76f8e1b37e626eaeaf96c5488150
Author: Philipp Winter
Date: Fri Aug 14 11:29:01 2020 -0700
Use format string for consistency.
Thanks to Rob for pointing this out.
---
onionperf/model.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/onionperf/model.py b/onionperf/model.py
index 3bfe35f..d45763e 100644
--- a/onionperf/model.py
+++ b/onionperf/model.py
@@ -118,7 +118,7 @@ class TorperfModel(GeneratableTGenModel):
else:
# one-shot mode, i.e., end after configured number of transfers
g.add_node("end",
- count=str(self.config.num_transfers))
+ count="%d" % self.config.num_transfers)
# check for end condition after every transfer
g.add_edge("stream", "end")
# if end condition not met, pause
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/master] Make models more configurable.
commit 7ef8c64833d41337d5c9cc5baaee2808092c9aad Author: Philipp Winter Date: Fri Jun 26 10:00:29 2020 -0700 Make models more configurable. This patch removes the --oneshot subcommand and replaces it with several new subcommands for OnionPerf's "measure" command: --tgen-start-pause (Initial pause before file transfers.) --tgen-num-transfers(Number of file transfers.) --tgen-intertransfer-pause (Pause in between file transfers.) --tgen-transfer-size(Size of each file transfer.) By default, OnionPerf continues to run in "continuous" mode. One can simulate oneshot mode by running onionperf with the following flags: onionperf measure --tgen-num-transfers=1 In addition to the above subcommands, this patch improves the code base by 1) adding a TGenConf class to hold TGen's configuration and by 2) adding a TGenModelConf class to hold TGen's traffic model. This fixes tpo/metrics/onionperf#33432. --- onionperf/measurement.py| 102 +++--- onionperf/model.py | 108 +++- onionperf/onionperf | 60 +++- onionperf/tests/test_measurement.py | 12 ++-- 4 files changed, 175 insertions(+), 107 deletions(-) diff --git a/onionperf/measurement.py b/onionperf/measurement.py index af1fa0d..e2d8d1c 100644 --- a/onionperf/measurement.py +++ b/onionperf/measurement.py @@ -15,6 +15,16 @@ from stem.control import Controller from stem.version import Version, Requirement, get_system_tor_version from stem import __version__ as stem_version +class TGenConf(object): +"""Represents a TGen configuration, for both client and server.""" +def __init__(self, listen_port=None, connect_ip=None, connect_port=None, tor_ctl_port=None, tor_socks_port=None): +self.listen_port = str(listen_port) +self.tor_ctl_port = tor_ctl_port +self.tor_socks_port = tor_socks_port +# TGen clients use connect_ip and connect_port. +self.connect_ip = connect_ip +self.connect_port = connect_port + # onionperf imports from . import analysis, monitor, model, util @@ -173,12 +183,11 @@ def logrotate_thread_task(writables, tgen_writable, torctl_writable, docroot, ni class Measurement(object): -def __init__(self, tor_bin_path, tgen_bin_path, datadir_path, privatedir_path, nickname, oneshot, additional_client_conf=None, torclient_conf_file=None, torserver_conf_file=None, single_onion=False): +def __init__(self, tor_bin_path, tgen_bin_path, datadir_path, privatedir_path, nickname, additional_client_conf=None, torclient_conf_file=None, torserver_conf_file=None, single_onion=False): self.tor_bin_path = tor_bin_path self.tgen_bin_path = tgen_bin_path self.datadir_path = datadir_path self.privatedir_path = privatedir_path -self.oneshot = oneshot self.nickname = nickname self.threads = None self.done_event = None @@ -190,20 +199,30 @@ class Measurement(object): self.torserver_conf_file = torserver_conf_file self.single_onion = single_onion -def run(self, do_onion=True, do_inet=True, client_tgen_listen_port=5, client_tgen_connect_ip='0.0.0.0', client_tgen_connect_port=8080, client_tor_ctl_port=59050, client_tor_socks_port=59000, - server_tgen_listen_port=8080, server_tor_ctl_port=59051, server_tor_socks_port=59001): +def run(self, do_onion=True, do_inet=True, tgen_model=None, tgen_client_conf=None, tgen_server_conf=None): ''' -only `server_tgen_listen_port` are "public" and need to be opened on the firewall. -if `client_tgen_connect_port` != `server_tgen_listen_port`, then you should have installed a forwarding rule in the firewall. +only `tgen_server_conf.listen_port` are "public" and need to be opened on the firewall. +if `tgen_client_conf.connect_port` != `tgen_server_conf.listen_port`, then you should have installed a forwarding rule in the firewall. all ports need to be unique though, and unique among multiple onionperf instances. here are some sane defaults: -client_tgen_listen_port=5, client_tgen_connect_port=8080, client_tor_ctl_port=59050, client_tor_socks_port=59000, -server_tgen_listen_port=8080, server_tor_ctl_port=59051, server_tor_socks_port=59001 +tgen_client_conf.listen_port=5, tgen_client_conf.connect_port=8080, tgen_client_conf.tor_ctl_port=59050, tgen_client_conf.tor_socks_port=59000, +tgen_server_conf.listen_port=8080, tgen_server_conf.tor_ctl_port=59051, tgen_server_conf.tor_socks_port=59001 ''' self.threads = [] self.done_event = threading.Event() +if tgen_client_conf is None: +tgen_client_conf = TGenConf(listen_port=5, +
[tor-commits] [onionperf/develop] Tweak --drop-guards switch.
commit 9d76ca4b3847a10c9565d12cf130933a1076297b
Author: Karsten Loesing
Date: Tue Sep 1 17:02:47 2020 +0200
Tweak --drop-guards switch.
This commit tweaks the recently added --drop-guards switch as follows:
- Guards are dropped right at startup and then every N hours.
Otherwise we might not receive the first round of GUARD NEW/UP
events. It's unclear why we don't receive those events, but finding
out might be time-consuming whereas dropping guards at startup is
basically free.
- Right after guards are dropped, circuit build timeouts are dropped,
too, if supported by the Tor version. If the Tor version does not
support this, there's going to be a warning, and the control log
will simply not contain BUILDTIMEOUT_SET events.
Still part of the reopened tpo/metrics/onionperf#33399.
---
CHANGELOG.md | 3 ++-
onionperf/monitor.py | 12 +++-
onionperf/onionperf | 2 +-
3 files changed, 10 insertions(+), 7 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a212842..e0253c5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,7 +1,8 @@
# Changes in version 0.7 - 2020-??-??
- Add `onionperf measure --drop-guards` parameter to use and drop
- guards after a given number of hours. Implements #33399.
+ guards and circuit build timeouts every given number of hours, if
+ supported by the Tor version. Implements #33399.
- Remove the `onionperf measure --oneshot` switch and replace it with
new switches `--tgen-pause-initial`, `--tgen-pause-between`,
`--tgen-transfer-size`, and `--tgen-num-transfers ` to further
diff --git a/onionperf/monitor.py b/onionperf/monitor.py
index b5e6390..886a9a0 100644
--- a/onionperf/monitor.py
+++ b/onionperf/monitor.py
@@ -57,21 +57,23 @@ class TorMonitor(object):
interval_count = 0
if newnym_interval_seconds is not None:
next_newnym = newnym_interval_seconds
-if drop_guards_interval_hours > 0:
-next_drop_guards = drop_guards_interval_hours * 3600
+next_drop_guards = 0
while done_ev is None or not done_ev.is_set():
# if self.filepath != '-' and
os.path.exists(self.filepath):
#with open(self.filepath, 'rb') as sizef:
#msg = "tor-ctl-logger[port={0}] logged {1} bytes
to {2}, press CTRL-C to quit".format(self.tor_ctl_port,
os.fstat(sizef.fileno()).st_size, self.filepath)
#logging.info(msg)
+if drop_guards_interval_hours > 0 and interval_count >=
next_drop_guards:
+next_drop_guards += drop_guards_interval_hours * 3600
+torctl.drop_guards()
+drop_timeouts_response = torctl.msg("DROPTIMEOUTS")
+if not drop_timeouts_response.is_ok():
+self.__log(self.writable, "[WARNING] unrecognized
command DROPTIMEOUTS in tor\n")
sleep(1)
interval_count += 1
if newnym_interval_seconds is not None and interval_count
>= next_newnym:
next_newnym += newnym_interval_seconds
torctl.signal(Signal.NEWNYM)
-if drop_guards_interval_hours > 0 and interval_count >=
next_drop_guards:
-next_drop_guards += drop_guards_interval_hours * 3600
-torctl.drop_guards()
except KeyboardInterrupt:
pass # the user hit ctrl+c
diff --git a/onionperf/onionperf b/onionperf/onionperf
index e6aa44a..d3d8e31 100755
--- a/onionperf/onionperf
+++ b/onionperf/onionperf
@@ -215,7 +215,7 @@ def main():
default=0)
measure_parser.add_argument('--drop-guards',
-help="""Use and drop guards every N > 0 hours, or do not use guards at
all if N = 0""",
+help="""Use and drop guards and circuit build timeouts every N > 0
hours, or do not use guards at all and never drop circuit build timeouts if N =
0""",
metavar="N", type=type_nonnegative_integer,
action="store", dest="drop_guards_interval_hours",
default=0)
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/develop] Merge remote-tracking branch 'karsten/task-33399-2' into develop
commit 8fe84702dea4b2e4640c0817e505e6c449e7b9a8 Merge: f496ede 9d76ca4 Author: Karsten Loesing Date: Tue Sep 1 22:28:52 2020 +0200 Merge remote-tracking branch 'karsten/task-33399-2' into develop CHANGELOG.md | 3 ++- onionperf/monitor.py | 12 +++- onionperf/onionperf | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/develop] Bump version to 0.7.
commit c8275b25e4afda9328634ec6be56ff46c7ee1cfe
Author: Karsten Loesing
Date: Tue Sep 1 22:30:12 2020 +0200
Bump version to 0.7.
---
CHANGELOG.md | 2 +-
setup.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e0253c5..80fb14d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,4 @@
-# Changes in version 0.7 - 2020-??-??
+# Changes in version 0.7 - 2020-09-01
- Add `onionperf measure --drop-guards` parameter to use and drop
guards and circuit build timeouts every given number of hours, if
diff --git a/setup.py b/setup.py
index 7fd7f85..d72b27a 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@ with open('requirements.txt') as f:
install_requires = f.readlines()
setup(name='OnionPerf',
- version='0.6',
+ version='0.7',
description='A utility to monitor, measure, analyze, and visualize the
performance of Tor and Onion Services',
author='Rob Jansen',
url='https://github.com/robgjansen/onionperf/',
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/master] Clarify base64 encoding of sha256 digest.
commit e58e6e1f2d1da5b60abca31fa9a263e3d3a3f4ec Author: Karsten Loesing Date: Fri Aug 28 10:56:15 2020 +0200 Clarify base64 encoding of sha256 digest. --- src/main/resources/web/jsps/collector.jsp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/resources/web/jsps/collector.jsp b/src/main/resources/web/jsps/collector.jsp index b738089..788128c 100644 --- a/src/main/resources/web/jsps/collector.jsp +++ b/src/main/resources/web/jsps/collector.jsp @@ -955,7 +955,8 @@ Index files use the following custom JSON data format that might still be extend "types": Descriptor types as found in @type annotations of contained descriptors. "first_published": Earliest publication timestamp of contained descriptors using pattern "-MM-DD HH:MM" in the UTC timezone. "last_published": Latest publication timestamp of contained descriptors using pattern "-MM-DD HH:MM" in the UTC timezone. -"sha256": SHA-256 digest of this file. +"sha256": SHA-256 digest of this file, encoded in Base64. + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/develop] Add issue number to change log.
commit f496edec6fec8adc0b4e5c0454989c2bc06312e3 Author: Karsten Loesing Date: Thu Aug 27 11:09:07 2020 +0200 Add issue number to change log. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b7fcb2..a212842 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ - Remove the `onionperf measure --oneshot` switch and replace it with new switches `--tgen-pause-initial`, `--tgen-pause-between`, `--tgen-transfer-size`, and `--tgen-num-transfers ` to further - configure the generated TGen model. + configure the generated TGen model. Implemets #33432. # Changes in version 0.6 - 2020-08-08 ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/develop] Rename variables for consistency and clarity.
commit 86f746c3c6ee6dc3be342f53fea90e6c02d39991
Author: Philipp Winter
Date: Fri Aug 14 10:49:09 2020 -0700
Rename variables for consistency and clarity.
initial_pause-> pause_initial
inter_transfer_pause -> pause_between
Thanks to Rob for the suggestion.
---
onionperf/model.py | 19 ---
onionperf/onionperf | 12 ++--
2 files changed, 14 insertions(+), 17 deletions(-)
diff --git a/onionperf/model.py b/onionperf/model.py
index b589249..3bfe35f 100644
--- a/onionperf/model.py
+++ b/onionperf/model.py
@@ -43,14 +43,14 @@ class TGenLoadableModel(TGenModel):
class TGenModelConf(object):
"""Represents a TGen traffic model configuration."""
-def __init__(self, initial_pause=0, num_transfers=1, transfer_size="5 MiB",
- continuous_transfers=False, inter_transfer_pause=5,
port=None, servers=[],
+def __init__(self, pause_initial=0, num_transfers=1, transfer_size="5 MiB",
+ continuous_transfers=False, pause_between=5, port=None,
servers=[],
socks_port=None):
-self.initial_pause = initial_pause
+self.pause_initial = pause_initial
+self.pause_between = pause_between
self.num_transfers = num_transfers
self.transfer_size = transfer_size
self.continuous_transfers = continuous_transfers
-self.inter_transfer_pause = inter_transfer_pause
self.port = port
self.servers = servers
self.socks_port = socks_port
@@ -98,20 +98,17 @@ class TorperfModel(GeneratableTGenModel):
loglevel="info",
heartbeat="1 minute")
-g.add_node("pause", time="%d seconds" % self.config.initial_pause)
-g.add_edge("start", "pause")
-
-# "One-shot mode," i.e., onionperf will stop after the given number of
-# iterations. The idea is:
-# start -> pause -> stream-1 -> pause-1 -> ... -> stream-n -> pause-n
-> end
+g.add_node("pause_initial",
+ time="%d seconds" % self.config.pause_initial)
g.add_node("stream",
sendsize="0",
recvsize=self.config.transfer_size,
timeout="15 seconds",
stallout="10 seconds")
g.add_node("pause_between",
- time="%d seconds" % self.config.inter_transfer_pause)
+ time="%d seconds" % self.config.pause_between)
+g.add_edge("start", "pause_initial")
g.add_edge("pause_initial", "stream")
# only add an end node if we need to stop
diff --git a/onionperf/onionperf b/onionperf/onionperf
index d95e691..a49982b 100755
--- a/onionperf/onionperf
+++ b/onionperf/onionperf
@@ -190,16 +190,16 @@ def main():
action="store", dest="tgenconnectport",
default=8080)
-measure_parser.add_argument('--tgen-start-pause',
+measure_parser.add_argument('--tgen-pause-initial',
help="""the number of seconds TGen should wait before walking through
its action graph""",
metavar="N", type=int,
-action="store", dest="tgenstartpause",
+action="store", dest="tgenpauseinitial",
default=5)
-measure_parser.add_argument('--tgen-intertransfer-pause',
+measure_parser.add_argument('--tgen-pause-between',
help="""the number of seconds TGen should wait in between two
transfers""",
metavar="N", type=int,
-action="store", dest="tgenintertransferpause",
+action="store", dest="tgenpausebetween",
default=300)
measure_parser.add_argument('--tgen-transfer-size',
@@ -377,11 +377,11 @@ def measure(args):
tor_ctl_port=server_tor_ctl_port,
tor_socks_port=server_tor_socks_port)
-tgen_model = TGenModelConf(initial_pause=args.tgenstartpause,
+tgen_model = TGenModelConf(pause_initial=args.tgenpauseinitial,
transfer_size=args.tgentransfersize,
num_transfers=args.tgennumtransfers,
continuous_transfers=args.tgennumtransfers
== 0,
-
inter_transfer_pause=args.tgenintertransferpause)
+ pause_between=args.tgenpausebetween)
meas = Measurement(args.torpath,
args.tgenpath,
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/develop] Let TGen client finish by itself in one-shot mode.
commit 959cf3689106189001a83c7e58dc40e10497a081
Author: Philipp Winter
Date: Fri Aug 7 14:48:58 2020 -0700
Let TGen client finish by itself in one-shot mode.
We tell TGen client to finish on its own by passing the count option to
the end node:
https://github.com/shadow/tgen/blob/master/doc/TGen-Options.md#end-options
This patch adds another argument to the function watchdog_thread_task(),
no_relaunch, which instructs the function to not re-launch its process
if it fails.
---
onionperf/measurement.py | 45 +++--
onionperf/model.py | 3 ++-
2 files changed, 25 insertions(+), 23 deletions(-)
diff --git a/onionperf/measurement.py b/onionperf/measurement.py
index e2d8d1c..d699292 100644
--- a/onionperf/measurement.py
+++ b/onionperf/measurement.py
@@ -50,10 +50,11 @@ def readline_thread_task(instream, q):
# wait for lines from stdout until the EOF
for line in iter(instream.readline, b''): q.put(line)
-def watchdog_thread_task(cmd, cwd, writable, done_ev, send_stdin,
ready_search_str, ready_ev):
+def watchdog_thread_task(cmd, cwd, writable, done_ev, send_stdin,
ready_search_str, ready_ev, no_relaunch):
-# launch or re-launch our sub process until we are told to stop
-# if we fail too many times in too short of time, give up and exit
+# launch or re-launch (or don't re-launch, if no_relaunch is set) our sub
+# process until we are told to stop if we fail too many times in too short
+# of time, give up and exit
failure_times = []
pause_time_seconds = 0
while done_ev.is_set() is False:
@@ -105,6 +106,10 @@ def watchdog_thread_task(cmd, cwd, writable, done_ev,
send_stdin, ready_search_s
subp.wait()
elif done_ev.is_set():
logging.info("command '{}' finished as expected".format(cmd))
+elif no_relaunch:
+logging.info("command '{}' finished on its own".format(cmd))
+# our command finished on its own. time to terminate.
+done_ev.set()
else:
logging.warning("command '{}' finished before
expected".format(cmd))
now = time.time()
@@ -284,15 +289,9 @@ class Measurement(object):
time.sleep(1)
while True:
if tgen_model.num_transfers:
-downloads = 0
-while True:
-downloads =
self.__get_download_count(tgen_client_writable.filename)
-time.sleep(1)
-if downloads >= tgen_model.num_transfers:
-logging.info("Onionperf has downloaded %d
files and will now shut down." % tgen_model.num_transfers)
-break
-else:
-continue
+# This function blocks until our TGen client process
+# terminated on its own.
+self.__wait_for_tgen_client()
break
if self.__is_alive():
@@ -366,7 +365,10 @@ class Measurement(object):
logging.info("Logging TGen {1} process output to
{0}".format(tgen_logpath, name))
tgen_cmd = "{0} {1}".format(self.tgen_bin_path, tgen_confpath)
-tgen_args = (tgen_cmd, tgen_datadir, tgen_writable, self.done_event,
None, None, None)
+# If we're running in "one-shot mode", TGen client will terminate on
+# its own and we don't need our watchdog to restart the process.
+no_relaunch = (name == "client" and tgen_model_conf.num_transfers)
+tgen_args = (tgen_cmd, tgen_datadir, tgen_writable, self.done_event,
None, None, None, no_relaunch)
tgen_watchdog = threading.Thread(target=watchdog_thread_task,
name="tgen_{0}_watchdog".format(name), args=tgen_args)
tgen_watchdog.start()
self.threads.append(tgen_watchdog)
@@ -464,7 +466,7 @@ WarnUnsafeSocks 0\nSafeLogging 0\nMaxCircuitDirtiness 60
seconds\nDataDirectory
tor_stdin_bytes = str_tools._to_bytes(tor_config)
tor_ready_str = "Bootstrapped 100"
tor_ready_ev = threading.Event()
-tor_args = (tor_cmd, tor_datadir, tor_writable, self.done_event,
tor_stdin_bytes, tor_ready_str, tor_ready_ev)
+tor_args = (tor_cmd, tor_datadir, tor_writable, self.done_event,
tor_stdin_bytes, tor_ready_str, tor_ready_ev, False)
tor_watchdog = threading.Thread(target=watchdog_thread_task,
name="tor_{0}_watchdog".format(name), args=tor_args)
tor_watchdog.start()
self.threads.append(tor_watchdog)
@@ -491,14 +493,13 @@ WarnUnsafeSocks 0\nSafeLogging 0\nMaxCircuitDirtiness 60
seconds\nDataDirectory
return tor_writable, torctl_writable
-def __get_download_count(self, tgen_logpath):
-count = 0
-if tgen_logpath is not None and os.path.exists(tgen_logpath):
-
[tor-commits] [onionperf/develop] Apply 1 suggestion(s) to 1 file(s)
commit a6aa4189e04ee1b05dbdb68b90f24e14bf3443ac
Author: Philipp Winter
Date: Fri Aug 14 16:47:04 2020 +
Apply 1 suggestion(s) to 1 file(s)
---
onionperf/model.py | 54 ++
1 file changed, 22 insertions(+), 32 deletions(-)
diff --git a/onionperf/model.py b/onionperf/model.py
index bdd5a53..b589249 100644
--- a/onionperf/model.py
+++ b/onionperf/model.py
@@ -104,40 +104,30 @@ class TorperfModel(GeneratableTGenModel):
# "One-shot mode," i.e., onionperf will stop after the given number of
# iterations. The idea is:
# start -> pause -> stream-1 -> pause-1 -> ... -> stream-n -> pause-n
-> end
-if self.config.num_transfers > 0:
-for i in range(self.config.num_transfers):
-g.add_node("stream-%d" % i,
- sendsize="0",
- recvsize=self.config.transfer_size,
- timeout="15 seconds",
- stallout="10 seconds")
-g.add_node("pause-%d" % i,
- time="%d seconds" %
self.config.inter_transfer_pause)
-
-g.add_edge("stream-%d" % i, "pause-%d" % i)
-if i > 0:
-g.add_edge("pause-%d" % (i-1), "stream-%d" % i)
-
+g.add_node("stream",
+ sendsize="0",
+ recvsize=self.config.transfer_size,
+ timeout="15 seconds",
+ stallout="10 seconds")
+g.add_node("pause_between",
+ time="%d seconds" % self.config.inter_transfer_pause)
+
+g.add_edge("pause_initial", "stream")
+
+# only add an end node if we need to stop
+if self.config.continuous_transfers:
+# continuous mode, i.e., no end node
+g.add_edge("stream", "pause_between")
+else:
+# one-shot mode, i.e., end after configured number of transfers
g.add_node("end",
count=str(self.config.num_transfers))
-g.add_edge("pause", "stream-0")
-g.add_edge("pause-%d" % (self.config.num_transfers - 1), "end")
-
-# Continuous mode, i.e., onionperf will not stop. The idea is:
-# start -> pause -> stream -> pause
-# ^ |
-# +---+
-elif self.config.continuous_transfers:
-g.add_node("stream",
- sendsize="0",
- recvsize=self.config.transfer_size,
- timeout="15 seconds",
- stallout="10 seconds")
-g.add_node("pause",
- time="%d seconds" % self.config.inter_transfer_pause)
-g.add_edge("pause", "stream")
-g.add_edge("stream", "pause")
-g.add_edge("pause", "stream")
+# check for end condition after every transfer
+g.add_edge("stream", "end")
+# if end condition not met, pause
+g.add_edge("end", "pause_between")
+
+g.add_edge("pause_between", "stream")
return g
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/develop] Make models more configurable.
commit 7ef8c64833d41337d5c9cc5baaee2808092c9aad Author: Philipp Winter Date: Fri Jun 26 10:00:29 2020 -0700 Make models more configurable. This patch removes the --oneshot subcommand and replaces it with several new subcommands for OnionPerf's "measure" command: --tgen-start-pause (Initial pause before file transfers.) --tgen-num-transfers(Number of file transfers.) --tgen-intertransfer-pause (Pause in between file transfers.) --tgen-transfer-size(Size of each file transfer.) By default, OnionPerf continues to run in "continuous" mode. One can simulate oneshot mode by running onionperf with the following flags: onionperf measure --tgen-num-transfers=1 In addition to the above subcommands, this patch improves the code base by 1) adding a TGenConf class to hold TGen's configuration and by 2) adding a TGenModelConf class to hold TGen's traffic model. This fixes tpo/metrics/onionperf#33432. --- onionperf/measurement.py| 102 +++--- onionperf/model.py | 108 +++- onionperf/onionperf | 60 +++- onionperf/tests/test_measurement.py | 12 ++-- 4 files changed, 175 insertions(+), 107 deletions(-) diff --git a/onionperf/measurement.py b/onionperf/measurement.py index af1fa0d..e2d8d1c 100644 --- a/onionperf/measurement.py +++ b/onionperf/measurement.py @@ -15,6 +15,16 @@ from stem.control import Controller from stem.version import Version, Requirement, get_system_tor_version from stem import __version__ as stem_version +class TGenConf(object): +"""Represents a TGen configuration, for both client and server.""" +def __init__(self, listen_port=None, connect_ip=None, connect_port=None, tor_ctl_port=None, tor_socks_port=None): +self.listen_port = str(listen_port) +self.tor_ctl_port = tor_ctl_port +self.tor_socks_port = tor_socks_port +# TGen clients use connect_ip and connect_port. +self.connect_ip = connect_ip +self.connect_port = connect_port + # onionperf imports from . import analysis, monitor, model, util @@ -173,12 +183,11 @@ def logrotate_thread_task(writables, tgen_writable, torctl_writable, docroot, ni class Measurement(object): -def __init__(self, tor_bin_path, tgen_bin_path, datadir_path, privatedir_path, nickname, oneshot, additional_client_conf=None, torclient_conf_file=None, torserver_conf_file=None, single_onion=False): +def __init__(self, tor_bin_path, tgen_bin_path, datadir_path, privatedir_path, nickname, additional_client_conf=None, torclient_conf_file=None, torserver_conf_file=None, single_onion=False): self.tor_bin_path = tor_bin_path self.tgen_bin_path = tgen_bin_path self.datadir_path = datadir_path self.privatedir_path = privatedir_path -self.oneshot = oneshot self.nickname = nickname self.threads = None self.done_event = None @@ -190,20 +199,30 @@ class Measurement(object): self.torserver_conf_file = torserver_conf_file self.single_onion = single_onion -def run(self, do_onion=True, do_inet=True, client_tgen_listen_port=5, client_tgen_connect_ip='0.0.0.0', client_tgen_connect_port=8080, client_tor_ctl_port=59050, client_tor_socks_port=59000, - server_tgen_listen_port=8080, server_tor_ctl_port=59051, server_tor_socks_port=59001): +def run(self, do_onion=True, do_inet=True, tgen_model=None, tgen_client_conf=None, tgen_server_conf=None): ''' -only `server_tgen_listen_port` are "public" and need to be opened on the firewall. -if `client_tgen_connect_port` != `server_tgen_listen_port`, then you should have installed a forwarding rule in the firewall. +only `tgen_server_conf.listen_port` are "public" and need to be opened on the firewall. +if `tgen_client_conf.connect_port` != `tgen_server_conf.listen_port`, then you should have installed a forwarding rule in the firewall. all ports need to be unique though, and unique among multiple onionperf instances. here are some sane defaults: -client_tgen_listen_port=5, client_tgen_connect_port=8080, client_tor_ctl_port=59050, client_tor_socks_port=59000, -server_tgen_listen_port=8080, server_tor_ctl_port=59051, server_tor_socks_port=59001 +tgen_client_conf.listen_port=5, tgen_client_conf.connect_port=8080, tgen_client_conf.tor_ctl_port=59050, tgen_client_conf.tor_socks_port=59000, +tgen_server_conf.listen_port=8080, tgen_server_conf.tor_ctl_port=59051, tgen_server_conf.tor_socks_port=59001 ''' self.threads = [] self.done_event = threading.Event() +if tgen_client_conf is None: +tgen_client_conf = TGenConf(listen_port=5, +
[tor-commits] [onionperf/develop] Make some tweaks to new TGen model.
commit b8f1e5c2695c097a7494f7975403664c0c833825
Author: Karsten Loesing
Date: Sun Aug 16 22:03:34 2020 +0200
Make some tweaks to new TGen model.
- Change timeout back to 270 seconds and stallout back to 0 seconds.
- Change initial pause to 300 seconds to keep default behavior
unchanged.
- Change model, so that pause_between starts in parallel to a stream,
not when the stream is completed. This is the same behavior as
before.
Also add a change log entry for all changes.
---
CHANGELOG.md| 7 +++
onionperf/model.py | 20
onionperf/onionperf | 2 +-
3 files changed, 16 insertions(+), 13 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0c4c4f2..ac6897b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,10 @@
+# Changes in version 0.7 - 2020-??-??
+
+ - Remove the `onionperf measure --oneshot` switch and replace it with
+ new switches `--tgen-pause-initial`, `--tgen-pause-between`,
+ `--tgen-transfer-size`, and `--tgen-num-transfers ` to further
+ configure the generated TGen model.
+
# Changes in version 0.6 - 2020-??-??
- Update to TGen 1.0.0, use TGenTools for parsing TGen log files, and
diff --git a/onionperf/model.py b/onionperf/model.py
index d45763e..fde587f 100644
--- a/onionperf/model.py
+++ b/onionperf/model.py
@@ -43,8 +43,8 @@ class TGenLoadableModel(TGenModel):
class TGenModelConf(object):
"""Represents a TGen traffic model configuration."""
-def __init__(self, pause_initial=0, num_transfers=1, transfer_size="5 MiB",
- continuous_transfers=False, pause_between=5, port=None,
servers=[],
+def __init__(self, pause_initial=300, num_transfers=1, transfer_size="5
MiB",
+ continuous_transfers=False, pause_between=300, port=None,
servers=[],
socks_port=None):
self.pause_initial = pause_initial
self.pause_between = pause_between
@@ -103,28 +103,24 @@ class TorperfModel(GeneratableTGenModel):
g.add_node("stream",
sendsize="0",
recvsize=self.config.transfer_size,
- timeout="15 seconds",
- stallout="10 seconds")
+ timeout="270 seconds",
+ stallout="0 seconds")
g.add_node("pause_between",
time="%d seconds" % self.config.pause_between)
g.add_edge("start", "pause_initial")
g.add_edge("pause_initial", "stream")
+g.add_edge("pause_initial", "pause_between")
+g.add_edge("pause_between", "stream")
+g.add_edge("pause_between", "pause_between")
# only add an end node if we need to stop
-if self.config.continuous_transfers:
-# continuous mode, i.e., no end node
-g.add_edge("stream", "pause_between")
-else:
+if not self.config.continuous_transfers:
# one-shot mode, i.e., end after configured number of transfers
g.add_node("end",
count="%d" % self.config.num_transfers)
# check for end condition after every transfer
g.add_edge("stream", "end")
-# if end condition not met, pause
-g.add_edge("end", "pause_between")
-
-g.add_edge("pause_between", "stream")
return g
diff --git a/onionperf/onionperf b/onionperf/onionperf
index a49982b..6a16da2 100755
--- a/onionperf/onionperf
+++ b/onionperf/onionperf
@@ -194,7 +194,7 @@ def main():
help="""the number of seconds TGen should wait before walking through
its action graph""",
metavar="N", type=int,
action="store", dest="tgenpauseinitial",
-default=5)
+default=300)
measure_parser.add_argument('--tgen-pause-between',
help="""the number of seconds TGen should wait in between two
transfers""",
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/develop] Use format string for consistency.
commit 1eea5e10700c76f8e1b37e626eaeaf96c5488150
Author: Philipp Winter
Date: Fri Aug 14 11:29:01 2020 -0700
Use format string for consistency.
Thanks to Rob for pointing this out.
---
onionperf/model.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/onionperf/model.py b/onionperf/model.py
index 3bfe35f..d45763e 100644
--- a/onionperf/model.py
+++ b/onionperf/model.py
@@ -118,7 +118,7 @@ class TorperfModel(GeneratableTGenModel):
else:
# one-shot mode, i.e., end after configured number of transfers
g.add_node("end",
- count=str(self.config.num_transfers))
+ count="%d" % self.config.num_transfers)
# check for end condition after every transfer
g.add_edge("stream", "end")
# if end condition not met, pause
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionperf/develop] Merge branch 'phw-enhancement-33432-3' into develop
commit c707674ba9b0d7931038c12bbe2d01585a88eb22
Merge: dfec0b8 b8f1e5c
Author: Karsten Loesing
Date: Thu Aug 27 10:38:21 2020 +0200
Merge branch 'phw-enhancement-33432-3' into develop
CHANGELOG.md| 4 ++
onionperf/measurement.py| 139 +++-
onionperf/model.py | 98 ++---
onionperf/onionperf | 60
onionperf/tests/test_measurement.py | 12 ++--
5 files changed, 185 insertions(+), 128 deletions(-)
diff --cc CHANGELOG.md
index c31a40e,ac6897b..4b7fcb2
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@@ -1,9 -1,11 +1,13 @@@
# Changes in version 0.7 - 2020-??-??
+ - Add `onionperf measure --drop-guards` parameter to use and drop
+ guards after a given number of hours. Implements #33399.
+ - Remove the `onionperf measure --oneshot` switch and replace it with
+new switches `--tgen-pause-initial`, `--tgen-pause-between`,
+`--tgen-transfer-size`, and `--tgen-num-transfers ` to further
+configure the generated TGen model.
-# Changes in version 0.6 - 2020-??-??
+# Changes in version 0.6 - 2020-08-08
- Update to TGen 1.0.0, use TGenTools for parsing TGen log files, and
update analysis results file version to 3.0. Implements #33974.
diff --cc onionperf/measurement.py
index 709fbc6,d699292..f198be0
--- a/onionperf/measurement.py
+++ b/onionperf/measurement.py
@@@ -173,7 -188,7 +188,7 @@@ def logrotate_thread_task(writables, tg
class Measurement(object):
- def __init__(self, tor_bin_path, tgen_bin_path, datadir_path,
privatedir_path, nickname, oneshot, additional_client_conf=None,
torclient_conf_file=None, torserver_conf_file=None, single_onion=False,
drop_guards_interval_hours=0):
-def __init__(self, tor_bin_path, tgen_bin_path, datadir_path,
privatedir_path, nickname, additional_client_conf=None,
torclient_conf_file=None, torserver_conf_file=None, single_onion=False):
++def __init__(self, tor_bin_path, tgen_bin_path, datadir_path,
privatedir_path, nickname, additional_client_conf=None,
torclient_conf_file=None, torserver_conf_file=None, single_onion=False,
drop_guards_interval_hours=0):
self.tor_bin_path = tor_bin_path
self.tgen_bin_path = tgen_bin_path
self.datadir_path = datadir_path
@@@ -189,13 -203,11 +203,12 @@@
self.torclient_conf_file = torclient_conf_file
self.torserver_conf_file = torserver_conf_file
self.single_onion = single_onion
+self.drop_guards_interval_hours = drop_guards_interval_hours
- def run(self, do_onion=True, do_inet=True, client_tgen_listen_port=5,
client_tgen_connect_ip='0.0.0.0', client_tgen_connect_port=8080,
client_tor_ctl_port=59050, client_tor_socks_port=59000,
- server_tgen_listen_port=8080, server_tor_ctl_port=59051,
server_tor_socks_port=59001):
+ def run(self, do_onion=True, do_inet=True, tgen_model=None,
tgen_client_conf=None, tgen_server_conf=None):
'''
- only `server_tgen_listen_port` are "public" and need to be opened on
the firewall.
- if `client_tgen_connect_port` != `server_tgen_listen_port`, then you
should have installed a forwarding rule in the firewall.
+ only `tgen_server_conf.listen_port` are "public" and need to be
opened on the firewall.
+ if `tgen_client_conf.connect_port` != `tgen_server_conf.listen_port`,
then you should have installed a forwarding rule in the firewall.
all ports need to be unique though, and unique among multiple
onionperf instances.
here are some sane defaults:
diff --cc onionperf/onionperf
index 641db70,6a16da2..e6aa44a
--- a/onionperf/onionperf
+++ b/onionperf/onionperf
@@@ -195,12 -190,30 +190,36 @@@ def main()
action="store", dest="tgenconnectport",
default=8080)
+ measure_parser.add_argument('--tgen-pause-initial',
+ help="""the number of seconds TGen should wait before walking through
its action graph""",
+ metavar="N", type=int,
+ action="store", dest="tgenpauseinitial",
+ default=300)
+
+ measure_parser.add_argument('--tgen-pause-between',
+ help="""the number of seconds TGen should wait in between two
transfers""",
+ metavar="N", type=int,
+ action="store", dest="tgenpausebetween",
+ default=300)
+
+ measure_parser.add_argument('--tgen-transfer-size',
+ help="""the size of the file transfer that TGen will perform (e.g.,
'5 MiB' or '10 KiB')""",
+ metavar="STRING", type=str,
+ action="store", dest="tgentransfersize",
+ default="5 MiB")
+
+ measure_parser.add_argument('--tgen-num-transfers',
