[tor-commits] [onionoo/master] Avoid unnecessary conversions to String.
commit 09765ef24569d72666843440ae2a7225c2b38ac5
Author: Karsten Loesing
Date: Wed Nov 20 12:17:56 2019 +0100
Avoid unnecessary conversions to String.
---
.../java/org/torproject/metrics/onionoo/docs/BandwidthStatus.java | 4 ++--
src/main/java/org/torproject/metrics/onionoo/docs/NodeStatus.java | 4 ++--
.../java/org/torproject/metrics/onionoo/updater/DescriptorQueue.java | 3 +--
.../java/org/torproject/metrics/onionoo/updater/LookupService.java| 2 +-
.../org/torproject/metrics/onionoo/server/ResourceServletTest.java| 4 ++--
5 files changed, 8 insertions(+), 9 deletions(-)
diff --git
a/src/main/java/org/torproject/metrics/onionoo/docs/BandwidthStatus.java
b/src/main/java/org/torproject/metrics/onionoo/docs/BandwidthStatus.java
index 6466b63..c9eddf1 100644
--- a/src/main/java/org/torproject/metrics/onionoo/docs/BandwidthStatus.java
+++ b/src/main/java/org/torproject/metrics/onionoo/docs/BandwidthStatus.java
@@ -173,12 +173,12 @@ public class BandwidthStatus extends Document {
for (long[] v : writeHistory.values()) {
sb.append("w ").append(DateTimeHelper.format(v[0])).append(" ")
.append(DateTimeHelper.format(v[1])).append(" ")
- .append(String.valueOf(v[2])).append("\n");
+ .append(v[2]).append("\n");
}
for (long[] v : readHistory.values()) {
sb.append("r ").append(DateTimeHelper.format(v[0])).append(" ")
.append(DateTimeHelper.format(v[1])).append(" ")
- .append(String.valueOf(v[2])).append("\n");
+ .append(v[2]).append("\n");
}
return sb.toString();
}
diff --git a/src/main/java/org/torproject/metrics/onionoo/docs/NodeStatus.java
b/src/main/java/org/torproject/metrics/onionoo/docs/NodeStatus.java
index 826db9e..1294bd5 100644
--- a/src/main/java/org/torproject/metrics/onionoo/docs/NodeStatus.java
+++ b/src/main/java/org/torproject/metrics/onionoo/docs/NodeStatus.java
@@ -724,11 +724,11 @@ public class NodeStatus extends Document {
sb.append("\t").append(this.dirPort).append("\t");
sb.append(StringUtils.join(this.getRelayFlags(), ","));
if (this.isRelay) {
- sb.append("\t").append(String.valueOf(this.consensusWeight));
+ sb.append("\t").append(this.consensusWeight);
sb.append("\t")
.append((this.countryCode != null ? this.countryCode : "??"));
sb.append("\t"); /* formerly used for storing host names */
- sb.append("\t").append(String.valueOf(this.lastRdnsLookup));
+ sb.append("\t").append(this.lastRdnsLookup);
sb.append("\t").append((this.defaultPolicy != null
? this.defaultPolicy : "null"));
sb.append("\t").append((this.portList != null ? this.portList : "null"));
diff --git
a/src/main/java/org/torproject/metrics/onionoo/updater/DescriptorQueue.java
b/src/main/java/org/torproject/metrics/onionoo/updater/DescriptorQueue.java
index 6cae37e..c609b2a 100644
--- a/src/main/java/org/torproject/metrics/onionoo/updater/DescriptorQueue.java
+++ b/src/main/java/org/torproject/metrics/onionoo/updater/DescriptorQueue.java
@@ -124,8 +124,7 @@ class DescriptorQueue {
for (Map.Entry e : excludedAndParsedFiles.entrySet()) {
String absolutePath = e.getKey();
long lastModifiedMillis = e.getValue();
-bw.write(String.valueOf(lastModifiedMillis) + " " + absolutePath
-+ "\n");
+bw.write(lastModifiedMillis + " " + absolutePath + "\n");
}
} catch (IOException e) {
log.error("Could not write history file '{}'. Not excluding descriptors "
diff --git
a/src/main/java/org/torproject/metrics/onionoo/updater/LookupService.java
b/src/main/java/org/torproject/metrics/onionoo/updater/LookupService.java
index 947c78c..63fcaa8 100644
--- a/src/main/java/org/torproject/metrics/onionoo/updater/LookupService.java
+++ b/src/main/java/org/torproject/metrics/onionoo/updater/LookupService.java
@@ -248,7 +248,7 @@ public class LookupService {
this.geoLite2AsnBlocksIpv4CsvFile.getAbsolutePath());
return lookupResults;
}
- String asNumber = "AS" + String.valueOf(Integer.parseInt(parts[1]));
+ String asNumber = "AS" + Integer.parseInt(parts[1]);
String asName = parts[2];
while (firstAddressNumber < startIpNum
&& firstAddressNumber != -1L) {
diff --git
a/src/test/java/org/torproject/metrics/onionoo/server/ResourceServletTest.java
b/src/test/java/org/torproject/metrics/onionoo/server/ResourceServletTest.java
index 7df56ea..9224620 100644
---
a/src/test/java/org/torproject/metrics/onionoo/server/ResourceServletTest.java
+++
b/src/test/java/org/torp
[tor-commits] [collector/master] Fix minor JavaDoc issue.
commit 38234a37884b091ac3b06e027bc2a92c019f6f11
Author: Karsten Loesing
Date: Wed Nov 20 11:47:48 2019 +0100
Fix minor JavaDoc issue.
---
src/main/java/org/torproject/metrics/collector/conf/Key.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/main/java/org/torproject/metrics/collector/conf/Key.java
b/src/main/java/org/torproject/metrics/collector/conf/Key.java
index d59438b..d0b8a5a 100644
--- a/src/main/java/org/torproject/metrics/collector/conf/Key.java
+++ b/src/main/java/org/torproject/metrics/collector/conf/Key.java
@@ -89,7 +89,7 @@ public enum Key {
private static Set keys;
/**
- * @param Class of key value.
+ * @param clazz Class of key value.
*/
Key(Class clazz) {
this.clazz = clazz;
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/master] Fix logging bug.
commit e25b39757cc464e0450f88e3779b9c987d003d58
Author: Karsten Loesing
Date: Wed Nov 20 11:54:46 2019 +0100
Fix logging bug.
---
src/main/java/org/torproject/metrics/collector/cron/Scheduler.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/main/java/org/torproject/metrics/collector/cron/Scheduler.java
b/src/main/java/org/torproject/metrics/collector/cron/Scheduler.java
index db10205..054f03f 100644
--- a/src/main/java/org/torproject/metrics/collector/cron/Scheduler.java
+++ b/src/main/java/org/torproject/metrics/collector/cron/Scheduler.java
@@ -58,7 +58,7 @@ public final class Scheduler implements ThreadFactory {
try {
gracePeriodMinutes = conf.getLong(Key.ShutdownGraceWaitMinutes);
} catch (ConfigurationException ce) {
- logger.warn("Cannot read grace period: {}", ce);
+ logger.warn("Cannot read grace period.", ce);
gracePeriodMinutes = 10L;
}
List> runOnceMains = new ArrayList<>();
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/master] Tweak DownloaderTest a bit.
commit 5d5d2eba7fb1c1fda3a0a16fe1cb5daf84f8ec47
Author: Karsten Loesing
Date: Wed Nov 20 12:00:15 2019 +0100
Tweak DownloaderTest a bit.
---
.../metrics/collector/downloader/DownloaderTest.java | 15 ++-
1 file changed, 6 insertions(+), 9 deletions(-)
diff --git
a/src/test/java/org/torproject/metrics/collector/downloader/DownloaderTest.java
b/src/test/java/org/torproject/metrics/collector/downloader/DownloaderTest.java
index 0617940..aef8c0c 100644
---
a/src/test/java/org/torproject/metrics/collector/downloader/DownloaderTest.java
+++
b/src/test/java/org/torproject/metrics/collector/downloader/DownloaderTest.java
@@ -15,7 +15,6 @@ import org.junit.BeforeClass;
import org.junit.Test;
import java.io.ByteArrayInputStream;
-import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.SocketTimeoutException;
import java.net.URL;
@@ -42,21 +41,19 @@ public class DownloaderTest {
*/
private static class HttpUrlStreamHandler extends URLStreamHandler {
-private Map connections = new HashMap();
+private Map connections = new HashMap<>();
@Override
-protected URLConnection openConnection(URL url) throws IOException {
- return connections.get(url);
+protected URLConnection openConnection(URL url) {
+ return this.connections.get(url);
}
private void resetConnections() {
- connections = new HashMap();
+ this.connections = new HashMap<>();
}
-private HttpUrlStreamHandler addConnection(URL url,
-URLConnection urlConnection) {
- connections.put(url, urlConnection);
- return this;
+private void addConnection(URL url, URLConnection urlConnection) {
+ this.connections.put(url, urlConnection);
}
}
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionoo/master] Make inner classes static.
commit bc0f0a6451372726fe4a018b7cecaf95c0f03709
Author: Karsten Loesing
Date: Wed Nov 20 12:19:53 2019 +0100
Make inner classes static.
---
.../org/torproject/metrics/onionoo/server/ResourceServletTest.java| 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/src/test/java/org/torproject/metrics/onionoo/server/ResourceServletTest.java
b/src/test/java/org/torproject/metrics/onionoo/server/ResourceServletTest.java
index 9224620..ac3f107 100644
---
a/src/test/java/org/torproject/metrics/onionoo/server/ResourceServletTest.java
+++
b/src/test/java/org/torproject/metrics/onionoo/server/ResourceServletTest.java
@@ -45,7 +45,7 @@ public class ResourceServletTest {
private static long TEST_TIME = DateTimeHelper.parse("2013-04-24 12:22:22");
- private class TestingHttpServletRequestWrapper
+ private static class TestingHttpServletRequestWrapper
extends HttpServletRequestWrapper {
private String requestUri;
private String queryString;
@@ -78,7 +78,7 @@ public class ResourceServletTest {
}
}
- private class TestingHttpServletResponseWrapper extends
+ private static class TestingHttpServletResponseWrapper extends
HttpServletResponseWrapper {
private TestingHttpServletResponseWrapper() {
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/master] Fix two JavaDocs issues with package-info.java.
commit 3bbc4db433979867cfcbac08eb7a698d4259a009
Author: Karsten Loesing
Date: Wed Nov 20 11:53:11 2019 +0100
Fix two JavaDocs issues with package-info.java.
---
.../java/org/torproject/metrics/collector/persist/package-info.java | 6 +++---
.../java/org/torproject/metrics/collector/sync/package-info.java| 3 +--
2 files changed, 4 insertions(+), 5 deletions(-)
diff --git
a/src/main/java/org/torproject/metrics/collector/persist/package-info.java
b/src/main/java/org/torproject/metrics/collector/persist/package-info.java
index 11d3939..75ec95e 100644
--- a/src/main/java/org/torproject/metrics/collector/persist/package-info.java
+++ b/src/main/java/org/torproject/metrics/collector/persist/package-info.java
@@ -1,11 +1,11 @@
/* Copyright 2016--2018 The Tor Project
* See LICENSE for licensing information */
-package org.torproject.metrics.collector.persist;
-
-/** This package containes decorating classes for descriptors that
+/** This package contains decorating classes for descriptors that
* simply determine the two storage paths based on the descriptor
* and further parameters like acquisition time.
* All special persistence classes extend
* {@code DescriptorPersistence}.
*/
+package org.torproject.metrics.collector.persist;
+
diff --git
a/src/main/java/org/torproject/metrics/collector/sync/package-info.java
b/src/main/java/org/torproject/metrics/collector/sync/package-info.java
index 08a6cdd..53e64b7 100644
--- a/src/main/java/org/torproject/metrics/collector/sync/package-info.java
+++ b/src/main/java/org/torproject/metrics/collector/sync/package-info.java
@@ -1,8 +1,6 @@
/* Copyright 2016--2018 The Tor Project
* See LICENSE for licensing information */
-package org.torproject.metrics.collector.sync;
-
/** This package coordinates syncing and merging the fetched data.
* The central class for this process is {@code SyncManager}, which
* coordinates download from other instances and merging the new data
@@ -10,4 +8,5 @@ package org.torproject.metrics.collector.sync;
* Storing data to the file system is facilitated by
* {@code SyncPersistence}.
*/
+package org.torproject.metrics.collector.sync;
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/master] Use StandardCharsets.US_ASCII instead of "US-ASCII".
commit 7338d79d1abcfd04857a10818d076d6a7917649b
Author: Karsten Loesing
Date: Wed Nov 20 11:49:54 2019 +0100
Use StandardCharsets.US_ASCII instead of "US-ASCII".
---
.../collector/bridgedescs/BridgeDescriptorParser.java| 3 ++-
.../collector/bridgedescs/BridgeSnapshotReader.java | 3 ++-
.../collector/bridgedescs/SanitizedBridgesWriter.java| 6 +++---
.../metrics/collector/relaydescs/ArchiveReader.java | 2 +-
.../relaydescs/CachedRelayDescriptorReader.java | 7 ---
.../collector/relaydescs/RelayDescriptorParser.java | 9 +
.../collector/relaydescs/ReferenceCheckerTest.java | 16
7 files changed, 25 insertions(+), 21 deletions(-)
diff --git
a/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeDescriptorParser.java
b/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeDescriptorParser.java
index 04460c8..778d187 100644
---
a/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeDescriptorParser.java
+++
b/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeDescriptorParser.java
@@ -9,6 +9,7 @@ import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
public class BridgeDescriptorParser {
@@ -33,7 +34,7 @@ public class BridgeDescriptorParser {
String authorityFingerprint) {
try {
BufferedReader br = new BufferedReader(new StringReader(
- new String(allData, "US-ASCII")));
+ new String(allData, StandardCharsets.US_ASCII)));
String line = br.readLine();
if (line == null) {
return;
diff --git
a/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeSnapshotReader.java
b/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeSnapshotReader.java
index 68ee28b..041807e 100644
---
a/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeSnapshotReader.java
+++
b/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeSnapshotReader.java
@@ -20,6 +20,7 @@ import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
@@ -139,7 +140,7 @@ public class BridgeSnapshotReader {
}
String fileDigest = Hex.encodeHexString(DigestUtils.sha1(
allData));
-String ascii = new String(allData, "US-ASCII");
+String ascii = new String(allData, StandardCharsets.US_ASCII);
BufferedReader br3 = new BufferedReader(new StringReader(
ascii));
String firstLine;
diff --git
a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
index 921396c..45f2a1a 100644
---
a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
+++
b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
@@ -470,7 +470,7 @@ public class SanitizedBridgesWriter extends CollecTorMain {
try {
DescriptorBuilder scrubbed = new DescriptorBuilder();
BufferedReader br = new BufferedReader(new StringReader(new String(
- data, "US-ASCII")));
+ data, StandardCharsets.US_ASCII)));
String line;
String mostRecentDescPublished = null;
byte[] fingerprintBytes = null;
@@ -682,7 +682,7 @@ public class SanitizedBridgesWriter extends CollecTorMain {
String masterKeyEd25519FromIdentityEd25519 = null;
DescriptorBuilder scrubbed = new DescriptorBuilder();
try (BufferedReader br = new BufferedReader(new StringReader(
-new String(data, "US-ASCII" {
+new String(data, StandardCharsets.US_ASCII {
scrubbed.append(Annotation.BridgeServer.toString());
String line;
String masterKeyEd25519 = null;
@@ -1112,7 +1112,7 @@ public class SanitizedBridgesWriter extends CollecTorMain
{
String masterKeyEd25519FromIdentityEd25519 = null;
try {
BufferedReader br = new BufferedReader(new StringReader(new String(
- data, "US-ASCII")));
+ data, StandardCharsets.US_ASCII)));
String line;
DescriptorBuilder scrubbed = null;
String hashedBridgeIdentity;
diff --git
a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java
b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java
index fee46c9..3326833 100644
---
a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java
+++
b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java
@@ -184,7
[tor-commits] [metrics-lib/master] Remove deprecation warning.
commit 25ea064dc8214385f2077706b2635f2da0d530fa
Author: Karsten Loesing
Date: Wed Nov 20 10:34:35 2019 +0100
Remove deprecation warning.
Class#newInstance is deprecated in Java 9 and higher, which doesn't
affect us yet in Java 8. But the suggested replacement already works
in Java 8, so that we can safely switch to that.
---
src/main/java/org/torproject/descriptor/DescriptorSourceFactory.java | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git
a/src/main/java/org/torproject/descriptor/DescriptorSourceFactory.java
b/src/main/java/org/torproject/descriptor/DescriptorSourceFactory.java
index e755e42..55fbcf3 100644
--- a/src/main/java/org/torproject/descriptor/DescriptorSourceFactory.java
+++ b/src/main/java/org/torproject/descriptor/DescriptorSourceFactory.java
@@ -146,10 +146,9 @@ public final class DescriptorSourceFactory {
+ ".");
}
object = ClassLoader.getSystemClassLoader().loadClass(clazzName)
- .newInstance();
+ .getDeclaredConstructor().newInstance();
log.info("Serving implementation {} for {}.", clazzName, type);
-} catch (ClassNotFoundException | InstantiationException
- | IllegalAccessException ex) {
+} catch (ReflectiveOperationException ex) {
throw new RuntimeException("Cannot load class "
+ clazzName + "for type " + type, ex);
}
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-lib/master] Fix logging.
commit 2614e6137ce7ddac40eb369175f3f24aad45d4ab
Author: Karsten Loesing
Date: Wed Nov 20 12:03:05 2019 +0100
Fix logging.
---
.../java/org/torproject/descriptor/index/DescriptorIndexCollector.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/src/main/java/org/torproject/descriptor/index/DescriptorIndexCollector.java
b/src/main/java/org/torproject/descriptor/index/DescriptorIndexCollector.java
index 9e053dc..c472da0 100644
---
a/src/main/java/org/torproject/descriptor/index/DescriptorIndexCollector.java
+++
b/src/main/java/org/torproject/descriptor/index/DescriptorIndexCollector.java
@@ -181,7 +181,7 @@ public class DescriptorIndexCollector implements
DescriptorCollector {
} catch (IOException ioe) {
log.warn("Cannot index local directory {} to skip any remote files that "
+ "already exist locally. Continuing with an either empty or "
- + "incomplete index of local files.", ioe);
+ + "incomplete index of local files.", localDir, ioe);
}
return locals;
}
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-lib/master] Make inner class static.
commit 26874eb8a9af953b907fbe90cd6a604626312f6d
Author: Karsten Loesing
Date: Wed Nov 20 12:08:57 2019 +0100
Make inner class static.
---
src/main/java/org/torproject/descriptor/impl/BandwidthFileImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/src/main/java/org/torproject/descriptor/impl/BandwidthFileImpl.java
b/src/main/java/org/torproject/descriptor/impl/BandwidthFileImpl.java
index 5d661e4..657b90e 100644
--- a/src/main/java/org/torproject/descriptor/impl/BandwidthFileImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/BandwidthFileImpl.java
@@ -165,7 +165,7 @@ public class BandwidthFileImpl extends DescriptorImpl
implements BandwidthFile {
}
}
- private class RelayLineImpl implements RelayLine {
+ private static class RelayLineImpl implements RelayLine {
private String nodeId;
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-lib/master] Simplify switch/case statement.
commit a18c70b33e89ac6a9c241860b15e46a1b54d0ea6
Author: Karsten Loesing
Date: Wed Nov 20 12:07:01 2019 +0100
Simplify switch/case statement.
---
src/main/java/org/torproject/descriptor/impl/ExitListImpl.java | 4
1 file changed, 4 deletions(-)
diff --git a/src/main/java/org/torproject/descriptor/impl/ExitListImpl.java
b/src/main/java/org/torproject/descriptor/impl/ExitListImpl.java
index bd5b2b6..b1ffa67 100644
--- a/src/main/java/org/torproject/descriptor/impl/ExitListImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/ExitListImpl.java
@@ -77,11 +77,7 @@ public class ExitListImpl extends DescriptorImpl implements
ExitList {
sb.append(line).append(ExitList.EOL);
break;
case "Published":
- sb.append(line).append(ExitList.EOL);
- break;
case "LastStatus":
- sb.append(line).append(ExitList.EOL);
- break;
case "ExitAddress":
sb.append(line).append(ExitList.EOL);
break;
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-lib/master] Use StandardCharsets.US_ASCII instead of "US-ASCII".
commit 75e1d31465aceb31c81f86575dc71abdab77095a
Author: Karsten Loesing
Date: Wed Nov 20 12:04:29 2019 +0100
Use StandardCharsets.US_ASCII instead of "US-ASCII".
---
.../java/org/torproject/descriptor/impl/ExitListImplTest.java| 9 +
.../org/torproject/descriptor/impl/TorperfResultImplTest.java| 8
2 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/src/test/java/org/torproject/descriptor/impl/ExitListImplTest.java
b/src/test/java/org/torproject/descriptor/impl/ExitListImplTest.java
index 0b53250..44a868b 100644
--- a/src/test/java/org/torproject/descriptor/impl/ExitListImplTest.java
+++ b/src/test/java/org/torproject/descriptor/impl/ExitListImplTest.java
@@ -12,6 +12,7 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
+import java.nio.charset.StandardCharsets;
import java.util.Map;
public class ExitListImplTest {
@@ -22,7 +23,7 @@ public class ExitListImplTest {
@Test
public void testAnnotatedInput() throws Exception {
ExitListImpl result = new ExitListImpl((tordnselAnnotation + input)
-.getBytes("US-ASCII"), null, fileName);
+.getBytes(StandardCharsets.US_ASCII), null, fileName);
assertEquals("Expected one annotation.", 1,
result.getAnnotations().size());
assertEquals(tordnselAnnotation.substring(0, 18),
@@ -38,7 +39,7 @@ public class ExitListImplTest {
public void testMultipleExitAddresses() throws Exception {
ExitListImpl result = new ExitListImpl(
(tordnselAnnotation + multiExitAddressInput)
-.getBytes("US-ASCII"), null, fileName);
+.getBytes(StandardCharsets.US_ASCII), null, fileName);
assertTrue("Unrecognized lines: " + result.getUnrecognizedLines(),
result.getUnrecognizedLines().isEmpty());
Map map = result.getEntries()
@@ -54,7 +55,7 @@ public class ExitListImplTest {
this.thrown.expect(DescriptorParseException.class);
this.thrown.expectMessage("Missing 'ExitAddress' line in exit list
entry.");
new ExitListImpl((tordnselAnnotation + insufficientInput[0])
-.getBytes("US-ASCII"), null, fileName);
+.getBytes(StandardCharsets.US_ASCII), null, fileName);
}
@Test
@@ -62,7 +63,7 @@ public class ExitListImplTest {
this.thrown.expect(DescriptorParseException.class);
this.thrown.expectMessage("Missing 'Published' line in exit list entry.");
new ExitListImpl((tordnselAnnotation + insufficientInput[1])
-.getBytes("US-ASCII"), null, fileName);
+.getBytes(StandardCharsets.US_ASCII), null, fileName);
}
private static final String tordnselAnnotation = "@type tordnsel 1.0\n";
diff --git
a/src/test/java/org/torproject/descriptor/impl/TorperfResultImplTest.java
b/src/test/java/org/torproject/descriptor/impl/TorperfResultImplTest.java
index 2af458f..b56490a 100644
--- a/src/test/java/org/torproject/descriptor/impl/TorperfResultImplTest.java
+++ b/src/test/java/org/torproject/descriptor/impl/TorperfResultImplTest.java
@@ -22,7 +22,7 @@ public class TorperfResultImplTest {
public void testAnnotatedInput() throws Exception {
TorperfResultImpl result = (TorperfResultImpl)
(TorperfResultImpl.parseTorperfResults((torperfAnnotation + input)
-.getBytes("US-ASCII"), null).get(0));
+.getBytes(StandardCharsets.US_ASCII), null).get(0));
assertEquals("Expected one annotation.", 1,
result.getAnnotations().size());
assertEquals(torperfAnnotation.substring(0, 17),
@@ -37,7 +37,7 @@ public class TorperfResultImplTest {
@Test
public void testPartiallyAnnotatedInput() throws Exception {
byte[] asciiBytes = (torperfAnnotation
-+ input + input + input).getBytes("US-ASCII");
++ input + input + input).getBytes(StandardCharsets.US_ASCII);
List result = TorperfResultImpl.parseTorperfResults(
asciiBytes, null);
assertEquals("Expected one annotation.", 1,
@@ -53,7 +53,7 @@ public class TorperfResultImplTest {
public void testAllAnnotatedInput() throws Exception {
byte[] asciiBytes = (torperfAnnotation + input
+ torperfAnnotation + input
-+ torperfAnnotation + input).getBytes("US-ASCII");
++ torperfAnnotation + input).getBytes(StandardCharsets.US_ASCII);
List result = TorperfResultImpl.parseTorperfResults(
asciiBytes, null);
assertEquals("Expected one annotation.", 1,
@@ -68,7 +68,7 @@ public class TorperfResultImplTest {
@Test
public void testTrailingNewlinesRetained() throws Exception {
byte[] asciiBytes = (torperfAnnotation + input
-+ torperfAnnotation + input).getBytes("US-ASCII");
++ torperfAnnotation + input).getBytes(StandardCharsets.US_ASCII);
List result = TorperfResultImpl.parseTorperfResults(
asciiBy
[tor-commits] [metrics-lib/master] Simplify method call.
commit 5af3f60993a5ccf68545ca83a485e966a66343a9
Author: Karsten Loesing
Date: Wed Nov 20 12:07:49 2019 +0100
Simplify method call.
---
src/main/java/org/torproject/descriptor/impl/KeyValueMap.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/main/java/org/torproject/descriptor/impl/KeyValueMap.java
b/src/main/java/org/torproject/descriptor/impl/KeyValueMap.java
index 08ad84e..6cb4680 100644
--- a/src/main/java/org/torproject/descriptor/impl/KeyValueMap.java
+++ b/src/main/java/org/torproject/descriptor/impl/KeyValueMap.java
@@ -20,7 +20,7 @@ public class KeyValueMap extends TreeMap {
private void putPair(String key, T value, String line, String listElement,
int keyLength) throws DescriptorParseException {
-if (this.keySet().contains(key)) {
+if (this.containsKey(key)) {
throw new DescriptorParseException("Line '" + line + "' contains "
+ "duplicate key '" + key + "'.");
}
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-lib/master] Remove unused suppression.
commit b3b12366411eaba769dfc95b4edd64c4e534487b
Author: Karsten Loesing
Date: Wed Nov 20 12:10:40 2019 +0100
Remove unused suppression.
---
src/main/java/org/torproject/descriptor/impl/ServerDescriptorImpl.java | 1 -
1 file changed, 1 deletion(-)
diff --git
a/src/main/java/org/torproject/descriptor/impl/ServerDescriptorImpl.java
b/src/main/java/org/torproject/descriptor/impl/ServerDescriptorImpl.java
index 3df90cc..995909c 100644
--- a/src/main/java/org/torproject/descriptor/impl/ServerDescriptorImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/ServerDescriptorImpl.java
@@ -844,7 +844,6 @@ public abstract class ServerDescriptorImpl extends
DescriptorImpl
}
@Override
- @SuppressWarnings("deprecation")
public List getHiddenServiceDirVersions() {
return this.hiddenServiceDir ? null : Collections.singletonList(2);
}
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/master] Fix logging bugs.
commit 1d7691bba248f2d0a1fb4cc8053f6f8a18388853
Author: Karsten Loesing
Date: Wed Nov 20 12:46:22 2019 +0100
Fix logging bugs.
---
src/main/java/org/torproject/metrics/stats/main/Main.java | 3 ++-
src/main/java/org/torproject/metrics/web/UpdateNews.java | 4 ++--
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/src/main/java/org/torproject/metrics/stats/main/Main.java
b/src/main/java/org/torproject/metrics/stats/main/Main.java
index 3dc2775..1dec6a3 100644
--- a/src/main/java/org/torproject/metrics/stats/main/Main.java
+++ b/src/main/java/org/torproject/metrics/stats/main/Main.java
@@ -68,7 +68,8 @@ public class Main {
} catch (NoSuchMethodException | IllegalAccessException
| InvocationTargetException e) {
log.warn("Caught an exception when invoking the main method of the {} "
-+ "module. Moving on to the next module, if available.", e);
++ "module. Moving on to the next module, if available.",
+module.getName(), e);
}
}
diff --git a/src/main/java/org/torproject/metrics/web/UpdateNews.java
b/src/main/java/org/torproject/metrics/web/UpdateNews.java
index abce4f1..92103fe 100644
--- a/src/main/java/org/torproject/metrics/web/UpdateNews.java
+++ b/src/main/java/org/torproject/metrics/web/UpdateNews.java
@@ -77,7 +77,7 @@ public class UpdateNews {
int space = desc.indexOf(" ", open);
int close = desc.indexOf("]", open);
if (open < 0 || space < 0 || close < 0) {
-log.warn("Cannot convert link in line {}. Exiting.");
+log.warn("Cannot convert link in line {}. Exiting.", line);
System.exit(1);
}
desc = desc.substring(0, open) + ""
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionoo/master] Remove disappearing family members.
commit 941ffd691b7ddaa7d4ae692746f056bcabdfc1d7
Author: Karsten Loesing
Date: Mon Nov 11 10:59:13 2019 +0100
Remove disappearing family members.
When a relay stops writing any family members into its server
descriptor we never cleared old family members but kept them forever.
We should do that.
Fixes #28314.
---
CHANGELOG.md | 7 +++
.../metrics/onionoo/updater/NodeDetailsStatusUpdater.java | 4 ++--
2 files changed, 9 insertions(+), 2 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b5c4c38..dc93ec6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,10 @@
+# Changes in version 7.0-1.2?.? - 2019-1?-??
+
+ * Medium changes
+ - Remove declared/alleged family members after they disappear from
+ server descriptors.
+
+
# Changes in version 7.0-1.21.1 - 2019-11-09
* Minor changes
diff --git
a/src/main/java/org/torproject/metrics/onionoo/updater/NodeDetailsStatusUpdater.java
b/src/main/java/org/torproject/metrics/onionoo/updater/NodeDetailsStatusUpdater.java
index a5f16e4..b25459d 100644
---
a/src/main/java/org/torproject/metrics/onionoo/updater/NodeDetailsStatusUpdater.java
+++
b/src/main/java/org/torproject/metrics/onionoo/updater/NodeDetailsStatusUpdater.java
@@ -172,8 +172,8 @@ public class NodeDetailsStatusUpdater implements
DescriptorListener,
detailsStatus.setExitPolicy(descriptor.getExitPolicyLines());
detailsStatus.setContact(descriptor.getContact());
detailsStatus.setPlatform(descriptor.getPlatform());
+SortedSet declaredFamily = new TreeSet<>();
if (descriptor.getFamilyEntries() != null) {
- SortedSet declaredFamily = new TreeSet<>();
for (String familyMember : descriptor.getFamilyEntries()) {
if (familyMember.startsWith("$") && familyMember.length() >= 41) {
declaredFamily.add(
@@ -182,8 +182,8 @@ public class NodeDetailsStatusUpdater implements
DescriptorListener,
declaredFamily.add(familyMember);
}
}
- this.declaredFamilies.put(fingerprint, declaredFamily);
}
+this.declaredFamilies.put(fingerprint, declaredFamily);
if (descriptor.getIpv6DefaultPolicy() != null
&& (descriptor.getIpv6DefaultPolicy().equals("accept")
|| descriptor.getIpv6DefaultPolicy().equals("reject"))
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/master] Document newly added index.json fields.
commit 4259a6dffe28ad26a3b50c079e155e57b40388a9 Author: Karsten Loesing Date: Mon Nov 11 15:06:25 2019 +0100 Document newly added index.json fields. Still part of #31204. --- src/main/resources/web/jsps/collector.jsp | 4 1 file changed, 4 insertions(+) diff --git a/src/main/resources/web/jsps/collector.jsp b/src/main/resources/web/jsps/collector.jsp index 414b23f..d56bdfe 100644 --- a/src/main/resources/web/jsps/collector.jsp +++ b/src/main/resources/web/jsps/collector.jsp @@ -913,6 +913,10 @@ Index files use the following custom JSON data format that might still be extend "path": Relative path of the file. "size": Size of the file in bytes. "last_modified": Timestamp when the file was last modified using pattern "-MM-DD HH:MM" in the UTC timezone. +"types": Descriptor types as found in @type annotations of contained descriptors. +"first_published": Earliest publication timestamp of contained descriptors using pattern "-MM-DD HH:MM" in the UTC timezone. +"last_published": Latest publication timestamp of contained descriptors using pattern "-MM-DD HH:MM" in the UTC timezone. +"sha256": SHA-256 digest of this file. ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/master] Bump version to 1.13.1-dev.
commit d3d63bb1d8eaaf2c33486d68181fbb199041c9a5 Author: Karsten Loesing Date: Mon Nov 11 10:19:39 2019 +0100 Bump version to 1.13.1-dev. --- CHANGELOG.md | 3 +++ build.xml| 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 174f2e1..5293207 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +# Changes in version 1.1?.? - 2019-1?-?? + + # Changes in version 1.13.1 - 2019-11-11 * Minor changes diff --git a/build.xml b/build.xml index e91dae3..c75b69e 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,7 @@ - + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/release] Prepare for 1.13.1 release.
commit 199d1bed2b337f769c3718c4473f2886e7ecfa06 Author: Karsten Loesing Date: Mon Nov 11 10:14:47 2019 +0100 Prepare for 1.13.1 release. --- CHANGELOG.md | 5 - build.xml| 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2edc44d..174f2e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ -# Changes in version 1.1?.? - 2019-1?-?? +# Changes in version 1.13.1 - 2019-11-11 + + * Minor changes + - Update to metrics-lib 2.9.1. # Changes in version 1.13.0 - 2019-11-09 diff --git a/build.xml b/build.xml index 9c72447..e91dae3 100644 --- a/build.xml +++ b/build.xml @@ -9,10 +9,10 @@ - + - + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/release] Bump version to 1.13.0-dev.
commit de8ab5f1e9ca1b0c995be6b858aad7ae3aa6f5cd Author: Karsten Loesing Date: Sat Nov 9 11:35:02 2019 +0100 Bump version to 1.13.0-dev. --- CHANGELOG.md | 3 +++ build.xml| 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ac0b3c..2edc44d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +# Changes in version 1.1?.? - 2019-1?-?? + + # Changes in version 1.13.0 - 2019-11-09 * Medium changes diff --git a/build.xml b/build.xml index c1c8699..9c72447 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,7 @@ - + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/master] Prepare for 1.13.1 release.
commit 199d1bed2b337f769c3718c4473f2886e7ecfa06 Author: Karsten Loesing Date: Mon Nov 11 10:14:47 2019 +0100 Prepare for 1.13.1 release. --- CHANGELOG.md | 5 - build.xml| 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2edc44d..174f2e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ -# Changes in version 1.1?.? - 2019-1?-?? +# Changes in version 1.13.1 - 2019-11-11 + + * Minor changes + - Update to metrics-lib 2.9.1. # Changes in version 1.13.0 - 2019-11-09 diff --git a/build.xml b/build.xml index 9c72447..e91dae3 100644 --- a/build.xml +++ b/build.xml @@ -9,10 +9,10 @@ - + - + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/master] Add BridgeDB metrics to CollecTor page.
commit f760f191e93bb0aa8d64bfadb64256f986bf1921 Author: Karsten Loesing Date: Sun Nov 10 20:08:11 2019 +0100 Add BridgeDB metrics to CollecTor page. Still part of #19332. --- src/main/resources/web/jsps/collector.jsp | 26 ++ 1 file changed, 26 insertions(+) diff --git a/src/main/resources/web/jsps/collector.jsp b/src/main/resources/web/jsps/collector.jsp index 3bb0768..414b23f 100644 --- a/src/main/resources/web/jsps/collector.jsp +++ b/src/main/resources/web/jsps/collector.jsp @@ -211,6 +211,15 @@ recent archive + + BridgeDB metrics + + + BridgeDB metrics + @type bridgedb-metrics 1.0 + recent + archive + @@ -831,6 +840,23 @@ These are available from a DirPort's Snowflake statistics containing aggregated information about snowflake proxies and snowflake clients as generated by the snowflake broker. + + + +BridgeDB metrics +# + + +BridgeDB metrics +@type bridgedb-metrics 1.0 + recent + archive +# + + + +BridgeDB metrics contain aggregated information about requests to the BridgeDB service. + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/master] Update news.json to version 351 of doc/MetricsTimeline.
commit 2a3c01588d937c9c06e766dd17dbc5ee12606e47
Author: Karsten Loesing
Date: Sun Nov 10 11:54:02 2019 +0100
Update news.json to version 351 of doc/MetricsTimeline.
---
src/main/resources/web/json/news.json | 612 --
1 file changed, 590 insertions(+), 22 deletions(-)
diff --git a/src/main/resources/web/json/news.json
b/src/main/resources/web/json/news.json
index 1a204e0..1c07ca4 100644
--- a/src/main/resources/web/json/news.json
+++ b/src/main/resources/web/json/news.json
@@ -540,6 +540,9 @@
"links" : [ {
"label" : "blog post",
"target" :
"https://blog.torproject.org/combined-flash-proxy-pyobfsproxy-browser-bundles;
+ }, {
+"label" : "ticket",
+"target" : "https://bugs.torproject.org/7824;
} ]
}, {
"start" : "2013-02-08",
@@ -2165,7 +2168,7 @@
"label" : "news article",
"target" : "https://www.rt.com/business/388502-ukraine-bans-vk-yandex/;
}, {
-"label" : "reddit post",
+"label" : "Reddit post",
"target" : "https://www.reddit.com/r/TOR/comments/6c9ig1;
}, {
"label" : "relay graph",
@@ -2807,6 +2810,19 @@
"target" :
"https://metrics.torproject.org/versions.html?start=2017-12-01=2018-03-01;
} ]
}, {
+ "start" : "2018-01-18",
+ "end" : "2018-02-24",
+ "protocols" : [ "snowflake" ],
+ "short_description" : "Outage of Snowflake broker caused by the disabling of
the Let's Encrypt TLS-SNI-01 challenge.",
+ "description" : "Outage of Snowflake broker caused by the disabling of the
Let's Encrypt TLS-SNI-01 challenge.",
+ "links" : [ {
+"label" : "ticket",
+"target" : "https://bugs.torproject.org/25345;
+ }, {
+"label" : "graph",
+"target" : "https://bugs.torproject.org/30693#comment:3;
+ } ]
+}, {
"start" : "2018-01-20",
"protocols" : [ "relay" ],
"short_description" : "Some more directory authorities upgrade to 0.3.2.9
which enforces new requirements for the exit flag",
@@ -3026,6 +3042,19 @@
"target" : "https://bugs.torproject.org/25688#comment:1;
} ]
}, {
+ "start" : "2018-04-02",
+ "end" : "2018-04-18",
+ "protocols" : [ "snowflake" ],
+ "short_description" : "Accidentally misconfigured the 3 proxy-go instances
that were meant to point to the new standalone broker so that they pointed to
the old App Engine broker.",
+ "description" : "Accidentally misconfigured the 3 proxy-go instances that
were meant to point to the new standalone broker so that they pointed to the
old App Engine broker.",
+ "links" : [ {
+"label" : "start",
+"target" : "https://bugs.torproject.org/25688#comment:1;
+ }, {
+"label" : "end",
+"target" : "https://bugs.torproject.org/22874#comment:10;
+ } ]
+}, {
"start" : "2018-04-05",
"protocols" : [ "ipv4", "ipv6" ],
"short_description" : "geoip and geoip6 databases updated to \"April 3 2018
Maxmind GeoLite2 Country\"",
@@ -3045,6 +3074,15 @@
} ]
}, {
"start" : "2018-04-16",
+ "protocols" : [ "snowflake" ],
+ "short_description" : "Moved the Snowflake broker from App Engine to a
standalone server.",
+ "description" : "Moved the Snowflake broker from App Engine to a standalone
server.",
+ "links" : [ {
+"label" : "comment",
+"target" : "https://bugs.torproject.org/22874#comment:9;
+ } ]
+}, {
+ "start" : "2018-04-16",
"end" : "2018-05-08",
"places" : [ "ru" ],
"short_description" : "Russia tries to block Telegram; blocks about 18
million IP addresses including some belonging to Amazon and Google cloud
services.",
@@ -3060,16 +3098,6 @@
"target" :
"https://metrics.torproject.org/userstats-bridge-country.html?start=2018-03-01=2018-07-15=ru;
} ]
}, {
- "start" : "2018-04-20",
- "end" : "2018-04-27",
- "protocols" : [ "onion" ],
- "short_description" : "The number of v2 onion services increases from 70k to
120k.",
- "description" : "The number of v2 onion services increases from 70k to
120k.",
- &quo
[tor-commits] [metrics-web/master] Bump version to 1.3.0-dev.
commit 4ae4da944dbfe8b50085b43c9ef9f764959d3867 Author: Karsten Loesing Date: Sat Nov 9 22:46:28 2019 +0100 Bump version to 1.3.0-dev. --- CHANGELOG.md | 3 +++ build.xml| 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5cccaab..fe74cf5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +# Changes in version 1.?.? - 2019-1?-?? + + # Changes in version 1.3.0 - 2019-11-09 * Medium changes diff --git a/build.xml b/build.xml index e08a44a..83e69e7 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,7 @@ - + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [exonerator/master] Bump version to 4.2.0-dev.
commit ec51d14df47ee9dae29a9e2cbfbc9a3ba55459f9 Author: Karsten Loesing Date: Sat Nov 9 22:45:32 2019 +0100 Bump version to 4.2.0-dev. --- CHANGELOG.md | 3 +++ build.xml| 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 632f622..9c0cf51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +# Changes in version 4.?.? - 2019-1?-?? + + # Changes in version 4.2.0 - 2019-11-09 * Medium changes diff --git a/build.xml b/build.xml index 38fee16..312f249 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,7 @@ - + https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Update to latest metrics-base and metrics-lib.
commit 3a663e8ff6b5b0c0757fe787cf92eaac91734ad5 Author: Karsten Loesing Date: Tue Sep 17 10:05:27 2019 +0200 Update to latest metrics-base and metrics-lib. This update includes an update of PostgreSQL JDBC driver version to 42.2.5. --- CHANGELOG.md| 1 + build.xml | 2 +- src/build | 2 +- src/submods/metrics-lib | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41bb5df..2147ed7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ versions resolved by Ivy are the same as in Debian stretch with few exceptions. - Remove Cobertura from the build process. + - Update PostgreSQL JDBC driver version to 42.2.5. # Changes in version 1.2.0 - 2018-08-25 diff --git a/build.xml b/build.xml index 038a82f..3e8048f 100644 --- a/build.xml +++ b/build.xml @@ -57,7 +57,7 @@ - + diff --git a/src/build b/src/build index 75cda35..0674958 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 75cda35ba7375ab0f5b12fa214620c3a29917097 +Subproject commit 0674958512c705255d4ec4631d3280068ce0a7af diff --git a/src/submods/metrics-lib b/src/submods/metrics-lib index 7e396f0..c324d11 16 --- a/src/submods/metrics-lib +++ b/src/submods/metrics-lib @@ -1 +1 @@ -Subproject commit 7e396f01ef6de4cb3cd7e2b7fc977b4af7b97e7f +Subproject commit c324d11035059bd9677af120dc6471aebebf980a ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Remove Cobertura from the build process.
commit a881d0f480fe8c4289c47a9e14fdcd016a2ade9e Author: Karsten Loesing Date: Wed Aug 14 11:47:41 2019 +0200 Remove Cobertura from the build process. --- CHANGELOG.md| 1 + src/build | 2 +- src/submods/metrics-lib | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c0f935..41bb5df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ copied to the `lib/` directory manually. Current dependency versions resolved by Ivy are the same as in Debian stretch with few exceptions. + - Remove Cobertura from the build process. # Changes in version 1.2.0 - 2018-08-25 diff --git a/src/build b/src/build index 256e619..75cda35 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 256e6192ad0066e48c6abb0ee9ac71714c7a3a0f +Subproject commit 75cda35ba7375ab0f5b12fa214620c3a29917097 diff --git a/src/submods/metrics-lib b/src/submods/metrics-lib index e172870..53b16d1 16 --- a/src/submods/metrics-lib +++ b/src/submods/metrics-lib @@ -1 +1 @@ -Subproject commit e1728709b5ed30dace2dd5b7e47a3e3c5e9e5736 +Subproject commit 53b16d192da4a7fa5c1987a89a60257061602882 ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Update documentation of bridge or_addresses.
commit a15b169a00c8a7c44e0cf73a8fa26572f195a189 Author: Karsten Loesing Date: Thu Oct 31 14:42:14 2019 +0100 Update documentation of bridge or_addresses. Fixes #32322. --- src/main/resources/web/jsps/onionoo.jsp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/resources/web/jsps/onionoo.jsp b/src/main/resources/web/jsps/onionoo.jsp index 0448b6f..05e9723 100644 --- a/src/main/resources/web/jsps/onionoo.jsp +++ b/src/main/resources/web/jsps/onionoo.jsp @@ -1852,7 +1852,8 @@ IP version the bridge uses and to detect whether the bridge changed its address. Sanitized IP addresses always change on the 1st of every month at 00:00:00 UTC, regardless of the bridge actually changing its IP address. -TCP ports are not sanitized. +TCP ports are sanitized in a similar way as IP addresses to hide the +location of bridges with unusual TCP ports. ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Update CollecTor page to include snowflake stats.
commit 412ac16376bb93e9be80b776a64435940f16681d Author: Karsten Loesing Date: Wed Aug 14 10:58:38 2019 +0200 Update CollecTor page to include snowflake stats. Implements #29461. --- src/main/resources/web/jsps/collector.jsp | 24 1 file changed, 24 insertions(+) diff --git a/src/main/resources/web/jsps/collector.jsp b/src/main/resources/web/jsps/collector.jsp index a05b7e0..42280a9 100644 --- a/src/main/resources/web/jsps/collector.jsp +++ b/src/main/resources/web/jsps/collector.jsp @@ -196,6 +196,15 @@ recent archive + + Snowflake Statistics + + + Snowflake Statistics + @type snowflake-stats 1.0 + recent + archive + @@ -792,6 +801,21 @@ These are available from a DirPort's + +Snowflake Statistics +# + + +Snowflake Statistics +@type snowflake-stats 1.0 + recent + archive +# + + + +Snowflake statistics containing aggregated information about snowflake proxies and snowflake clients as generated by the snowflake broker. + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Add "Tor Browser updates by release channel" graph.
commit cba1711544fbfea9e88ba4161f94b26231b28d5f
Author: Karsten Loesing
Date: Wed Oct 2 12:24:38 2019 +0200
Add "Tor Browser updates by release channel" graph.
Implements #31755.
---
src/main/R/rserver/rserve-init.R | 46 ++
src/main/resources/web.xml | 4 ++
src/main/resources/web/json/categories.json| 1 +
src/main/resources/web/json/metrics.json | 11 ++
.../resources/web/jsps/reproducible-metrics.jsp| 1 +
src/main/resources/web/jsps/stats.jsp | 22 +++
6 files changed, 85 insertions(+)
diff --git a/src/main/R/rserver/rserve-init.R b/src/main/R/rserver/rserve-init.R
index af74ef7..5eac4d7 100644
--- a/src/main/R/rserver/rserve-init.R
+++ b/src/main/R/rserver/rserve-init.R
@@ -1399,6 +1399,52 @@ plot_webstats_tb_locale <- function(start_p, end_p,
path_p) {
ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}
+prepare_webstats_tb_channel <- function(start_p = NULL, end_p = NULL) {
+ read_csv(file = paste(stats_dir, "webstats.csv", sep = ""),
+ col_types = cols(
+log_date = col_date(format = ""),
+request_type = col_factor(levels = NULL),
+platform = col_skip(),
+channel = col_factor(levels = NULL),
+locale = col_skip(),
+incremental = col_skip(),
+count = col_double())) %>%
+filter(if (!is.null(start_p)) log_date >= as.Date(start_p) else TRUE) %>%
+filter(if (!is.null(end_p)) log_date <= as.Date(end_p) else TRUE) %>%
+filter(request_type %in% c("tbup", "tbur")) %>%
+filter(channel %in% c("a", "r")) %>%
+group_by(log_date, channel, request_type) %>%
+summarize(count = sum(count)) %>%
+dcast(log_date + channel ~ request_type, value.var = "count") %>%
+rename(date = log_date, update_pings = tbup, update_requests = tbur)
+}
+
+plot_webstats_tb_channel <- function(start_p, end_p, path_p) {
+ prepare_webstats_tb_channel(start_p, end_p) %>%
+gather(request_type, count, -c(date, channel)) %>%
+unite("request_type_channel", request_type, channel) %>%
+mutate(request_type_channel = factor(request_type_channel,
+ levels = c("update_pings_r", "update_pings_a",
+ "update_requests_r", "update_requests_a"),
+ labels = c("Update pings (stable)", "Update pings (alpha)",
+ "Update requests (stable)", "Update requests (alpha)"))) %>%
+ungroup() %>%
+complete(date = full_seq(date, period = 1),
+ nesting(request_type_channel)) %>%
+ggplot(aes(x = date, y = count)) +
+geom_point() +
+geom_line() +
+scale_x_date(name = "", breaks = custom_breaks,
+ labels = custom_labels, minor_breaks = custom_minor_breaks) +
+scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) +
+facet_grid(request_type_channel ~ ., scales = "free_y") +
+theme(strip.text.y = element_text(angle = 0, hjust = 0, size = rel(1.5)),
+ strip.background = element_rect(fill = NA)) +
+ggtitle("Tor Browser updates by release channel") +
+labs(caption = copyright_notice)
+ ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
+}
+
prepare_webstats_tm <- function(start_p = NULL, end_p = NULL) {
read_csv(file = paste(stats_dir, "webstats.csv", sep = ""),
col_types = cols(
diff --git a/src/main/resources/web.xml b/src/main/resources/web.xml
index 2c68fd7..10d12ee 100644
--- a/src/main/resources/web.xml
+++ b/src/main/resources/web.xml
@@ -51,6 +51,7 @@
/webstats-tb.html
/webstats-tb-platform.html
/webstats-tb-locale.html
+/webstats-tb-channel.html
/webstats-tm.html
/relays-ipv6.html
/bridges-ipv6.html
@@ -182,6 +183,9 @@
/webstats-tb-locale.png
/webstats-tb-locale.pdf
/webstats-tb-locale.csv
+/webstats-tb-channel.png
+/webstats-tb-channel.pdf
+/webstats-tb-channel.csv
/webstats-tm.png
/webstats-tm.pdf
/webstats-tm.csv
diff --git a/src/main/resources/web/json/categories.json
b/src/main/resources/web/json/categories.json
index 89742bc..3771631 100644
--- a/src/main/resources/web/json/categories.json
+++ b/src/main/resources/web/json/categories.json
@@ -88,6 +88,7 @@
"webstats-tb",
"webstats-tb-platform",
"webstats-tb-locale",
+ "webstats-tb-channel",
"webstats-tm"
]
}
diff --git a/src/main/resources/web/json/metrics.json
b/src/main/resources/web/json/metrics.json
index e6bab04..54ce78b 100644
--- a/src/main/resources/web/json/metrics.json
+++ b/src/main/resources/web/json/metrics.json
@@ -416,6 +416,17 @@
]
},
[tor-commits] [metrics-web/release] Update to metrics-lib 2.9.1 and ExoneraTor 4.2.0.
commit 9d34311e53389493e68d5b885da9928454807044 Author: Karsten Loesing Date: Sat Nov 9 20:38:51 2019 +0100 Update to metrics-lib 2.9.1 and ExoneraTor 4.2.0. --- CHANGELOG.md| 1 + build.xml | 4 ++-- src/submods/metrics-lib | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2147ed7..b8a4ed0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ few exceptions. - Remove Cobertura from the build process. - Update PostgreSQL JDBC driver version to 42.2.5. + - Update to metrics-lib 2.9.1 and ExoneraTor 4.2.0. # Changes in version 1.2.0 - 2018-08-25 diff --git a/build.xml b/build.xml index 959d069..3efe4bc 100644 --- a/build.xml +++ b/build.xml @@ -10,8 +10,8 @@ - - + + diff --git a/src/submods/metrics-lib b/src/submods/metrics-lib index 8763af4..87d998c 16 --- a/src/submods/metrics-lib +++ b/src/submods/metrics-lib @@ -1 +1 @@ -Subproject commit 8763af44465445b2257f9740d4ce341f8334eead +Subproject commit 87d998c542688c7e465486de24ef07c1cbece0b0 ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Add @type annotation for HS v3.
commit 66a657078b697e719cd9f5863fef62bdcb4d890a Author: Karsten Loesing Date: Mon Nov 4 12:24:38 2019 +0100 Add @type annotation for HS v3. Also update to latest metrics-base/-lib. Implements #31481. --- src/build | 2 +- src/main/resources/web/jsps/collector.jsp | 18 ++ src/submods/metrics-lib | 2 +- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/build b/src/build index 62a964a..eb16cb3 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 62a964a7702086ca6ba0e2307809d4f983163b47 +Subproject commit eb16cb359db41722e6089bafb1e26808df4338df diff --git a/src/main/resources/web/jsps/collector.jsp b/src/main/resources/web/jsps/collector.jsp index 102bcab..3bb0768 100644 --- a/src/main/resources/web/jsps/collector.jsp +++ b/src/main/resources/web/jsps/collector.jsp @@ -152,6 +152,11 @@ @type hidden-service-descriptor 1.0 + + Hidden Service Descriptors v3 + @type hidden-service-descriptor-3 1.0 + + BridgeDB's Bridge Pool Assignments @@ -553,6 +558,19 @@ Despite the version number being 1.0, these descriptors are part of the version 2 hidden service protocol. +Hidden Service Descriptors v3 +@type hidden-service-descriptor-3 1.0 +# + + + +Hidden service descriptors contain details required to connect to version +3 hidden service. +Despite the version number being 1.0, these descriptors are part of the +https://gitweb.torproject.org/torspec.git/tree/rend-spec-v3.txt;>version +3 hidden service protocol. + + BridgeDB's Bridge Pool Assignments # diff --git a/src/submods/metrics-lib b/src/submods/metrics-lib index 8e2f671..8763af4 16 --- a/src/submods/metrics-lib +++ b/src/submods/metrics-lib @@ -1 +1 @@ -Subproject commit 8e2f67107f1e6add0db3fa209cb89c3b69f7bf06 +Subproject commit 8763af44465445b2257f9740d4ce341f8334eead ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Use ant tasks to fetch metrics dependencies.
commit c30b25acbd32c5c6ff7c23c47062caf131d4a783
Author: Karsten Loesing
Date: Mon Sep 23 15:28:41 2019 +0200
Use ant tasks to fetch metrics dependencies.
Implements part of #31649.
---
.gitlab-ci.yml | 11 ++-
build.xml | 14 ++
src/build | 2 +-
src/submods/metrics-lib | 2 +-
4 files changed, 18 insertions(+), 11 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d10fcb8..ba2f6c1 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,7 +1,5 @@
variables:
GIT_STRATEGY: clone
- METRICS_LIB_VERSION: "2.6.2"
- EXONERATOR_VERSION: "4.1.0"
JAVA_TOOL_OPTIONS: "-Dfile.encoding=UTF-8"
stages:
@@ -16,13 +14,8 @@ test:
- git submodule init
- git submodule update
- mkdir lib
- - mkdir tmp
- - pushd tmp
- - curl
https://dist.torproject.org/metrics-lib/$METRICS_LIB_VERSION/metrics-lib-$METRICS_LIB_VERSION.tar.gz
| tar xzf -
- - curl
https://dist.torproject.org/exonerator/$EXONERATOR_VERSION/exonerator-$EXONERATOR_VERSION.tar.gz
| tar xzf -
- - popd
- - mv
tmp/metrics-lib-$METRICS_LIB_VERSION/generated/dist/metrics-lib-$METRICS_LIB_VERSION-thin.jar
lib
- - mv
tmp/exonerator-$EXONERATOR_VERSION/generated/dist/exonerator-$EXONERATOR_VERSION-thin.jar
lib
+ - ant fetch-metrics-lib
+ - ant fetch-exonerator
- ant -lib /usr/share/java resolve
- ant test
- ant checks
diff --git a/build.xml b/build.xml
index 3e8048f..959d069 100644
--- a/build.xml
+++ b/build.xml
@@ -308,6 +308,20 @@
+
+
+https://dist.torproject.org/exonerator/${exoneratorversion}/exonerator-${exoneratorversion}.tar.gz;
+ dest="${downloadedlibs}"/>
+
+
+
+
+
+
+
+
diff --git a/src/build b/src/build
index 0674958..07c2a00 16
--- a/src/build
+++ b/src/build
@@ -1 +1 @@
-Subproject commit 0674958512c705255d4ec4631d3280068ce0a7af
+Subproject commit 07c2a00c27f0d536223f8b5a61fc91e60eb524d4
diff --git a/src/submods/metrics-lib b/src/submods/metrics-lib
index c324d11..c1581dd 16
--- a/src/submods/metrics-lib
+++ b/src/submods/metrics-lib
@@ -1 +1 @@
-Subproject commit c324d11035059bd9677af120dc6471aebebf980a
+Subproject commit c1581dd8ca26798dc90ea2189ca41636b50e1e0d
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Adds explanation for first-hop build times overhead
commit 6a11d7ac3c372a99156be1ecf10c69f5afe3b6a3 Author: Ana Custura Date: Sat Jul 6 11:41:50 2019 +0100 Adds explanation for first-hop build times overhead --- src/main/resources/web/json/metrics.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/web/json/metrics.json b/src/main/resources/web/json/metrics.json index bfcde22..08e8c19 100644 --- a/src/main/resources/web/json/metrics.json +++ b/src/main/resources/web/json/metrics.json @@ -289,7 +289,7 @@ "id": "onionperf-buildtimes", "title": "Circuit build times", "type": "Graph", -"description": "This graph shows build times of circuits used for downloading static files of different sizes over Tor. The graph shows the range of measurements from first to third quartile, and highlights the median. The slowest and fastest quarter of measurements are omitted from the graph.", +"description": "This graph shows build times of circuits used for downloading static files of different sizes over Tor. We disable Entry Guards in the Tor client configuration used by OnionPerf, to ensure a new guard is selected for each circuit we measure. In comparison, standard clients will reuse connections to a small number of long-term entry guards, and do not see the latency introduced by establishing new guard connections. As a result, first hop build times appear higher for these measurements compared to those commonly observed in Tor clients. The graph shows the range of measurements from first to third quartile, and highlights the median. The slowest and fastest quarter of measurements are omitted from the graph.", "function": "onionperf_buildtimes", "parameters": [ "start", ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Prepare for 1.3.0 release.
commit d4581513c4ca31a007df243c10e697600d2b99e6 Author: Karsten Loesing Date: Sat Nov 9 21:43:49 2019 +0100 Prepare for 1.3.0 release. --- CHANGELOG.md | 2 +- build.xml| 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8a4ed0..5cccaab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changes in version 1.3.0 - 2018-??-?? +# Changes in version 1.3.0 - 2019-11-09 * Medium changes - Start downloading and processing votes. diff --git a/build.xml b/build.xml index 3efe4bc..e08a44a 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,7 @@ - + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Update to latest metrics-lib.
commit 37190035e688cc0a313bb06e10324c8fc609a0b6 Author: Karsten Loesing Date: Mon Sep 16 16:39:37 2019 +0200 Update to latest metrics-lib. --- src/submods/metrics-lib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/submods/metrics-lib b/src/submods/metrics-lib index 53b16d1..7e396f0 16 --- a/src/submods/metrics-lib +++ b/src/submods/metrics-lib @@ -1 +1 @@ -Subproject commit 53b16d192da4a7fa5c1987a89a60257061602882 +Subproject commit 7e396f01ef6de4cb3cd7e2b7fc977b4af7b97e7f ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Make userstats-combined.csv order platform independent.
commit 03d6a1a0305548d7c5b548feb905cb5c9525d266 Author: Karsten Loesing Date: Fri Mar 8 15:19:22 2019 +0100 Make userstats-combined.csv order platform independent. Turns out that we didn't specify the sorting order of userstats-combined.csv. However, different platforms produced consistently different outputs. Let's just define sort order to make the output deterministic, even across platforms. --- src/main/sql/clients/init-userstats.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/sql/clients/init-userstats.sql b/src/main/sql/clients/init-userstats.sql index cf2b620..467f8aa 100644 --- a/src/main/sql/clients/init-userstats.sql +++ b/src/main/sql/clients/init-userstats.sql @@ -761,5 +761,5 @@ CREATE OR REPLACE VIEW combined AS SELECT AND a.date < current_date - 1 -- Order results. - ORDER BY date DESC; + ORDER BY date, node, country, LOWER(transport); ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Re-add recent bridge pool assignments.
commit 7ccae10c5c171d363d907dd74736d51d89725748 Author: Karsten Loesing Date: Thu Sep 19 16:30:16 2019 +0200 Re-add recent bridge pool assignments. --- src/main/resources/web/jsps/collector.jsp | 9 +++-- 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/main/resources/web/jsps/collector.jsp b/src/main/resources/web/jsps/collector.jsp index 42280a9..102bcab 100644 --- a/src/main/resources/web/jsps/collector.jsp +++ b/src/main/resources/web/jsps/collector.jsp @@ -158,7 +158,8 @@ Bridge Pool Assignments @type bridge-pool-assignment 1.0 - archive + recent + archive TorDNSEL's Exit Lists @@ -573,6 +574,7 @@ statistical analysis. Bridge Pool Assignments @type bridge-pool-assignment 1.0 + recent archive # @@ -613,11 +615,6 @@ bridge-pool-assignment 2011-03-13 14:38:03 [...] - -As of December 8, 2014, bridge pool assignment files are no longer -archived. - - TorDNSEL's Exit Lists # ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Use Ivy for resolving external dependencies.
commit 844fcbe9f75551dc4af8b40e6a96483ffa9522ce Author: Karsten Loesing Date: Tue Aug 6 17:32:51 2019 +0200 Use Ivy for resolving external dependencies. Includes a metrics-lib upgrade to 2.6.2 and an upgrade to the latest metrics-base commit 256e619. Implements part of #31326. --- CHANGELOG.md| 9 +++ build.xml | 63 - src/build | 2 +- src/submods/metrics-lib | 2 +- 4 files changed, 37 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 028df01..3c0f935 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,15 @@ - Add Apache Commons Math 3.6.1 as dependency. - Extend ipv6servers module to generate servers part of legacy module. + - Use Ivy for resolving external dependencies rather than relying + on files found in Debian stable packages. Requires installing Ivy + (using `apt-get install ivy`, `brew install ivy`, or similar) and + running `ant resolve` (or `ant -lib /usr/share/java resolve`). + Retrieved files are then copied to the `lib/` directory, except + for dependencies on other metrics libraries that still need to be + copied to the `lib/` directory manually. Current dependency + versions resolved by Ivy are the same as in Debian stretch with + few exceptions. # Changes in version 1.2.0 - 2018-08-25 diff --git a/build.xml b/build.xml index 0c9fa0e..b4a27e5 100644 --- a/build.xml +++ b/build.xml @@ -4,12 +4,13 @@ ]> - + - + - - + + - + - + - - - - - - - - - - + + + + + + + + + + - - + + - - - + + + - - - - + + + + @@ -138,22 +139,10 @@ - - - - - - - - - - diff --git a/src/build b/src/build index ed48192..256e619 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit ed4819215aa60cc8935f955d9404f30e83dd892b +Subproject commit 256e6192ad0066e48c6abb0ee9ac71714c7a3a0f diff --git a/src/submods/metrics-lib b/src/submods/metrics-lib index 7032e73..e172870 16 --- a/src/submods/metrics-lib +++ b/src/submods/metrics-lib @@ -1 +1 @@ -Subproject commit 7032e73a6471f3a4d7782ef65fc3d4cf00bb0ac1 +Subproject commit e1728709b5ed30dace2dd5b7e47a3e3c5e9e5736 ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Replace "brackets" with "parentheses"
commit 7cfb22744fc9a7801bf5df171ff6cbcbd80c0b97 Author: Iain R. Learmonth Date: Wed Jun 5 13:30:07 2019 +0100 Replace "brackets" with "parentheses" Fixes: #29486 --- src/main/resources/web/templates/rs/search/do.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/web/templates/rs/search/do.html b/src/main/resources/web/templates/rs/search/do.html index 40745a6..1fe06b4 100644 --- a/src/main/resources/web/templates/rs/search/do.html +++ b/src/main/resources/web/templates/rs/search/do.html @@ -137,7 +137,7 @@ <% }); %> -The number shown in brackets is the total effective +The number shown in parentheses is the total effective family size for the relay including the relay itself. A size of 1 indicates that the relay does not have any other effective family members. Note that bridge relays do not advertise family members in their descriptors and so there ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Adds guidelines for new metrics page
commit 66704ba064c13f6349722edd8d63f93b5d2244ec
Author: Iain R. Learmonth
Date: Wed Jul 24 14:54:19 2019 +0100
Adds guidelines for new metrics page
Fixes: #29315
---
.../metrics/web/MetricsGuidelinesServlet.java | 25 +++
src/main/resources/web.xml | 11 +
src/main/resources/web/jsps/metrics-guidelines.jsp | 223 +
src/main/resources/web/jsps/sources.jsp| 6 +
4 files changed, 265 insertions(+)
diff --git
a/src/main/java/org/torproject/metrics/web/MetricsGuidelinesServlet.java
b/src/main/java/org/torproject/metrics/web/MetricsGuidelinesServlet.java
new file mode 100644
index 000..41dae13
--- /dev/null
+++ b/src/main/java/org/torproject/metrics/web/MetricsGuidelinesServlet.java
@@ -0,0 +1,25 @@
+/* Copyright 2018 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.metrics.web;
+
+import java.io.IOException;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+public class MetricsGuidelinesServlet extends AnyServlet {
+
+ private static final long serialVersionUID = 6099009779662419291L;
+
+ @Override
+ public void doGet(HttpServletRequest request,
+ HttpServletResponse response) throws IOException, ServletException {
+
+request.setAttribute("categories", this.categories);
+request.getRequestDispatcher("WEB-INF/metrics-guidelines.jsp")
+.forward(request, response);
+ }
+}
+
diff --git a/src/main/resources/web.xml b/src/main/resources/web.xml
index 045dd2e..f92813a 100644
--- a/src/main/resources/web.xml
+++ b/src/main/resources/web.xml
@@ -218,6 +218,17 @@
+MetricsGuidelinesServlet
+
+ org.torproject.metrics.web.MetricsGuidelinesServlet
+
+
+
+MetricsGuidelinesServlet
+/metrics-guidelines.html
+
+
+
NewsServlet
org.torproject.metrics.web.NewsServlet
diff --git a/src/main/resources/web/jsps/metrics-guidelines.jsp
b/src/main/resources/web/jsps/metrics-guidelines.jsp
new file mode 100644
index 000..0f06fe5
--- /dev/null
+++ b/src/main/resources/web/jsps/metrics-guidelines.jsp
@@ -0,0 +1,223 @@
+<%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core; %>
+<%@ taglib prefix="fn" uri="http://java.sun.com/jsp/jstl/functions; %>
+
+
+
+
+
+
+
+Home
+Sources
+Guidelines for getting your data into Tor Metrics
+
+
+
+
+
+Guidelines for getting your data into Tor Metrics
+#
+
+Scope and preliminaries
+
+This document provides guidelines to authors and operators of tools that
+ collect data about the publicly deployed Tor network that would like to
+ contribute data, or allow data to be contributed easily by others using
+ the tool, to Tor Metrics.
+
+This document does not discuss how to ensure measurements are safe, for this
+ refer to the Research Safety Board Guidelines [0] and
Guidelines for
+ Performing Safe Measurement on the Internet [1].
+
+What data belongs on Tor Metrics?
+
+
+If it happens in the public deployed Tor network it likely belongs on Tor
+ Metrics.
+If it happens for a short term only, like for a research project, it's
+ unlikely worth the effort to have Tor Metrics archive, publish, aggregate,
+ and visualize it. In this case you should collect the data yourself (keeping
+ in mind research ethics!), and we can later talk about linking to it or even
+ using it as external data.
+If your data is a combination of existing data on Tor Metrics plus maybe
+ external data, we shouldn't add it, either. In such a case we should rather
+ talk about extending our services towards what your service does, if that
+ makes sense.
+
+
+What data do you want to see on Tor Metrics?
+
+This section aims to help you organize your thoughts before making a request
+ to the Metrics team. It might be that there are good reasons that something is
+ not done in one of the preferred ways, but ideally data collection tools can
+ be written with this guidelines in mind.
+
+
+What is your data about? Is it about servers or users or both? Is it
+ passively gathered or actively measured or both?
+This will help us to decide how we might present the data on Tor Metrics
+ and perhaps which other datasets we have that might benefit from being
+ combined with the new dataset.
+Is there a way for you to aggregate the data before you hand it over to us?
+ Of course this requires more thinking upfront, but it's a great way to
ensure
+ not to give out too sensitive data to us or anyone else. It's not always
+ possible or even useful to aggregate data and discard the original data,
+ though. Two examples:
+ Relays count how many clients download the consensus from them and from
+ which country they connect. When 24 hours have passed, they include the
+ count by country in their next extra-info descriptor. This is aggregated
+ data. The obviously more sensitive, non-aggregated variant
[tor-commits] [metrics-web/release] Tweak the new guidelines a bit.
commit 9955072a6cf65e7012c317ccd8ebc616a211364f Author: Karsten Loesing Date: Mon Jul 29 10:25:22 2019 +0200 Tweak the new guidelines a bit. --- src/main/resources/web/jsps/metrics-guidelines.jsp | 12 ++-- 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/resources/web/jsps/metrics-guidelines.jsp b/src/main/resources/web/jsps/metrics-guidelines.jsp index 0f06fe5..eff0336 100644 --- a/src/main/resources/web/jsps/metrics-guidelines.jsp +++ b/src/main/resources/web/jsps/metrics-guidelines.jsp @@ -50,7 +50,7 @@ This section aims to help you organize your thoughts before making a request to the Metrics team. It might be that there are good reasons that something is not done in one of the preferred ways, but ideally data collection tools can - be written with this guidelines in mind. + be written with these guidelines in mind. What is your data about? Is it about servers or users or both? Is it @@ -76,8 +76,8 @@ Is the data you're planning to give us too sensitive? If so, can you sanitize it yourself before giving it to us (we can help you with that), or does the sanitizing need to happen on our side (we should still involve you in this - case)? -There are currently cases where Tor Metrics performs the sanitization of + case)? + There are currently cases where Tor Metrics performs the sanitization of data before archiving, but the preferred system would sanitize the data as close to the source as possible to minimize the possibility that sensitive data could be leaked. @@ -97,9 +97,9 @@ data, maybe. And if we need to include fancy crypto libraries in order to process your data, then for sure. Any intuitions you have about possible difficulties would be good to know, even if things turn out to be easier in - the end. -As far as possible, use simple formats for providing data. The Tor Directory - Protocol meta-format [2, §1.2] is a simple format for which we already have + the end. + As far as possible, use simple formats for providing data. The Tor Directory + Protocol meta-format [2, 1.2] is a simple format for which we already have parsers. Without good reason, do not serialize to formats such as YAML, TOML, etc. as this would require adding a new parsing library into Tor Metrics just to parse the new data. ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Fix generated metrics-lib JavaDocs.
commit bba56f6a536c34e858b1e78256e3209a5d27cd7e
Author: Karsten Loesing
Date: Wed Aug 7 21:33:34 2019 +0200
Fix generated metrics-lib JavaDocs.
---
build.xml | 1 +
1 file changed, 1 insertion(+)
diff --git a/build.xml b/build.xml
index b4a27e5..d8c0c07 100644
--- a/build.xml
+++ b/build.xml
@@ -210,6 +210,7 @@
dir="${basedir}/src/submods/metrics-lib"
failonerror="true" >
+
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Upgrade to latest metrics-base and -lib.
commit e2940260a1879d3a4354e826108366746b10a243 Author: Karsten Loesing Date: Wed Oct 2 14:35:54 2019 +0200 Upgrade to latest metrics-base and -lib. --- src/build | 2 +- src/submods/metrics-lib | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/build b/src/build index 07c2a00..62a964a 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 07c2a00c27f0d536223f8b5a61fc91e60eb524d4 +Subproject commit 62a964a7702086ca6ba0e2307809d4f983163b47 diff --git a/src/submods/metrics-lib b/src/submods/metrics-lib index c1581dd..8e2f671 16 --- a/src/submods/metrics-lib +++ b/src/submods/metrics-lib @@ -1 +1 @@ -Subproject commit c1581dd8ca26798dc90ea2189ca41636b50e1e0d +Subproject commit 8e2f67107f1e6add0db3fa209cb89c3b69f7bf06 ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Update news.json to version 308 of doc/MetricsTimeline.
commit ad7ed81857aab28f85e7c0f5f63286fff7230af8
Author: Karsten Loesing
Date: Fri Jan 11 11:42:42 2019 +0100
Update news.json to version 308 of doc/MetricsTimeline.
---
src/main/resources/web/json/news.json | 13 +
1 file changed, 13 insertions(+)
diff --git a/src/main/resources/web/json/news.json
b/src/main/resources/web/json/news.json
index 26bf78a..1a204e0 100644
--- a/src/main/resources/web/json/news.json
+++ b/src/main/resources/web/json/news.json
@@ -3468,6 +3468,19 @@
"target" :
"https://media.ccc.de/v/35c3-9964-cat_mouse_evading_the_censors_in_2018#t=1670;
} ]
}, {
+ "start" : "2019-01-07",
+ "end" : "2018-01-08",
+ "places" : [ "ga" ],
+ "short_description" : "Internet shutdown in Gabon following a coup attempt.",
+ "description" : "Internet shutdown in Gabon following a coup attempt.",
+ "links" : [ {
+"label" : "OONI report",
+"target" : "https://ooni.torproject.org/post/gabon-internet-disruption/;
+ }, {
+"label" : "NetBlocks report",
+"target" :
"https://netblocks.org/reports/evidence-of-gabon-full-internet-shutdown-coup-attempt-dQ8oo18n;
+ } ]
+}, {
"start" : "2016-02-24",
"places" : [ "tm" ],
"protocols" : [ "" ],
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Update to latest metrics-base and metrics-lib.
commit 84105ea10be7932b9cc11807991f0bda39ae2d75 Author: Karsten Loesing Date: Mon Jul 29 10:26:43 2019 +0200 Update to latest metrics-base and metrics-lib. --- src/build | 2 +- src/submods/metrics-lib | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/build b/src/build index e639c69..ed48192 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit e639c697e9e94c6dbb26e946e5247c20a62c0661 +Subproject commit ed4819215aa60cc8935f955d9404f30e83dd892b diff --git a/src/submods/metrics-lib b/src/submods/metrics-lib index e723c06..7032e73 16 --- a/src/submods/metrics-lib +++ b/src/submods/metrics-lib @@ -1 +1 @@ -Subproject commit e723c065b764ecfbb3bb96d4c491e67398b7f21b +Subproject commit 7032e73a6471f3a4d7782ef65fc3d4cf00bb0ac1 ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Adapt "About Tor" links to redesigned Tor website.
commit 31c4f88ebba9ad327594d4db47fa88fd7b5f6bb7 Author: Karsten Loesing Date: Mon May 13 17:22:04 2019 +0200 Adapt "About Tor" links to redesigned Tor website. Reported by nusenu, changes suggested by irl. Fixes #30415. --- build.xml | 2 +- src/main/resources/web/properties/ExoneraTor.properties| 7 +++ src/main/resources/web/properties/ExoneraTor_de.properties | 7 +++ src/main/resources/web/properties/ExoneraTor_fr.properties | 7 +++ src/main/resources/web/properties/ExoneraTor_ro.properties | 7 +++ src/main/resources/web/properties/ExoneraTor_sv.properties | 7 +++ 6 files changed, 16 insertions(+), 21 deletions(-) diff --git a/build.xml b/build.xml index 254a71f..0c9fa0e 100644 --- a/build.xml +++ b/build.xml @@ -10,7 +10,7 @@ - + diff --git a/src/main/resources/web/properties/ExoneraTor.properties b/src/main/resources/web/properties/ExoneraTor.properties index 936754a..05bb0cb 100644 --- a/src/main/resources/web/properties/ExoneraTor.properties +++ b/src/main/resources/web/properties/ExoneraTor.properties @@ -43,10 +43,9 @@ technicaldetails.exit.yes=Yes technicaldetails.exit.no=No permanentlink.heading=Permanent link footer.abouttor.heading=About Tor -footer.abouttor.body.text=Tor is an international software project to anonymize Internet traffic by %s. Therefore, if you see traffic from a Tor relay, this traffic usually originates from someone using Tor, rather than from the relay operator. The Tor Project and Tor relay operators have no records of the traffic that passes over the network and therefore cannot provide any information about its origin. Be sure to %s, and don't hesitate to %s for more information. -footer.abouttor.body.link1=encrypting packets and sending them through a series of hops before they reach their destination -footer.abouttor.body.link2=learn more about Tor -footer.abouttor.body.link3=contact The Tor Project, Inc. +footer.abouttor.body.text=Tor is an international software project to anonymize Internet traffic by encrypting packets and sending them through a series of hops before they reach their destination. Therefore, if you see traffic from a Tor relay, this traffic usually originates from someone using Tor, rather than from the relay operator. The Tor Project and Tor relay operators have no records of the traffic that passes over the network and therefore cannot provide any information about its origin. Be sure to %s, and don't hesitate to %s for more information. +footer.abouttor.body.link1=learn more about Tor +footer.abouttor.body.link2=contact The Tor Project, Inc. footer.aboutexonerator.heading=About ExoneraTor footer.aboutexonerator.body=The ExoneraTor service maintains a database of IP addresses that have been part of the Tor network. It answers the question whether there was a Tor relay running on a given IP address on a given date. ExoneraTor may store more than one IP address per relay if relays use a different IP address for exiting to the Internet than for registering in the Tor network, and it stores whether a relay permitted transit of Tor traffic to the open Internet at that time. footer.language.name=English diff --git a/src/main/resources/web/properties/ExoneraTor_de.properties b/src/main/resources/web/properties/ExoneraTor_de.properties index fbe8464..13b150b 100644 --- a/src/main/resources/web/properties/ExoneraTor_de.properties +++ b/src/main/resources/web/properties/ExoneraTor_de.properties @@ -43,10 +43,9 @@ technicaldetails.exit.yes=Ja technicaldetails.exit.no=Nein permanentlink.heading=Dauerhafter Link footer.abouttor.heading=\u00dcber Tor -footer.abouttor.body.text=Tor ist ein internationales Softwareprojekt um Internetverbindungen zu anonymisieren indem %s. Wenn Sie eine Internetverbindung sehen, die von einem Tor-Server kommt, stammt diese in der Regel von jemandem, der Tor benutzt, und nicht vom Betreiber des Tor-Servers selbst. Weder The Tor Project, Inc. noch die Tor-Server-Betreiber besitzen Aufzeichnungen \u00fcber die Internetverbindungen im Tor-Netzwerk und k\u00f6nnen daher keine Auskunft \u00fcber den Ursprung der Internetverbindung geben. Mehr Informationen \u00fcber Tor erhalten Sie %s. The Tor Project, Inc. steht Ihnen jederzeit f\u00fcr %s zur Verf\u00fcgung. -footer.abouttor.body.link1=Datenpakete verschl\u00fcsselt und \u00fcber eine Reihe von Stationen geleitet werden bevor diese ihr Ziel erreichen -footer.abouttor.body.link2=hier -footer.abouttor.body.link3=Fragen +footer.abouttor.body.text=Tor ist ein internationales Softwareprojekt um Internetverbindungen zu anonymisieren indem Datenpakete verschl\u00fcsselt und \u00fcber eine Reihe von Stationen geleitet werden bevor diese ihr Ziel erreichen. Wenn Sie eine Internetverbindung sehen, die von einem Tor-Server kommt, stammt diese in der Regel von jemandem, der Tor benutzt, und nicht vom Betreib
[tor-commits] [metrics-web/release] Add OnionPerf throughput graph.
commit d4452b38183e50d20c29bad6fff51bae58ebde0c
Author: Karsten Loesing
Date: Sun Jun 2 15:56:50 2019 +0200
Add OnionPerf throughput graph.
We calculate throughput from the time between receiving 0.5 and 1 MiB
of a response, which obviously excludes any measurements with
responses smaller than 1 MiB. From the FILESIZE and DATAPERC* fields
we can compute the number of milliseconds that have elapsed between
receiving bytes 524,288 and 1,048,576, which is a total of 524,288
bytes or 4,194,304 bits. We divide the value 4,194,304 by this time
difference to obtain throughput in bits per millisecond which happens
to be the same value as the number of kilobits per second.
Implements #29772.
---
src/main/R/rserver/rserve-init.R | 40 +++
.../torproject/metrics/stats/onionperf/Main.java | 30 +++
src/main/resources/web.xml | 4 ++
src/main/resources/web/json/categories.json| 3 +-
src/main/resources/web/json/metrics.json | 12 ++
.../resources/web/jsps/reproducible-metrics.jsp| 9 -
src/main/resources/web/jsps/stats.jsp | 26 +
src/main/sql/onionperf/init-onionperf.sql | 45 ++
src/submods/metrics-lib| 2 +-
9 files changed, 168 insertions(+), 3 deletions(-)
diff --git a/src/main/R/rserver/rserve-init.R b/src/main/R/rserver/rserve-init.R
index 88aa5b9..f74fd03 100644
--- a/src/main/R/rserver/rserve-init.R
+++ b/src/main/R/rserver/rserve-init.R
@@ -697,6 +697,46 @@ plot_onionperf_latencies <- function(start_p, end_p,
server_p, path_p) {
ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}
+prepare_onionperf_throughput <- function(start_p = NULL, end_p = NULL,
+server_p = NULL) {
+ read_csv(file = paste(stats_dir, "onionperf-throughput.csv", sep = ""),
+ col_types = cols(
+date = col_date(format = ""),
+source = col_character(),
+server = col_character(),
+low = col_double(),
+q1 = col_double(),
+md = col_double(),
+q3 = col_double(),
+high = col_double())) %>%
+filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
+filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
+filter(if (!is.null(server_p)) server == server_p else TRUE)
+}
+
+plot_onionperf_throughput <- function(start_p, end_p, server_p, path_p) {
+ prepare_onionperf_throughput(start_p, end_p, server_p) %>%
+complete(date = full_seq(date, period = 1), nesting(source)) %>%
+ggplot(aes(x = date, ymin = q1 / 1000, ymax = q3 / 1000, fill = source)) +
+geom_ribbon(alpha = 0.5) +
+geom_line(aes(y = md / 1000, colour = source), size = 0.75) +
+geom_line(aes(y = high / 1000, colour = source), size = 0.375) +
+geom_line(aes(y = low / 1000, colour = source), size = 0.375) +
+scale_x_date(name = "", breaks = custom_breaks,
+ labels = custom_labels, minor_breaks = custom_minor_breaks) +
+scale_y_continuous(name = "", labels = unit_format(unit = "Mbps"),
+ limits = c(0, NA)) +
+scale_fill_hue(name = "Source") +
+scale_colour_hue(name = "Source") +
+facet_grid(source ~ ., scales = "free", space = "free") +
+ggtitle(paste("Throughput when downloading from", server_p, "server")) +
+labs(caption = copyright_notice) +
+theme(legend.position = "none",
+ strip.text.y = element_text(angle = 0, hjust = 0),
+ strip.background = element_rect(fill = NA))
+ ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
+}
+
prepare_connbidirect <- function(start_p = NULL, end_p = NULL) {
read_csv(file = paste(stats_dir, "connbidirect2.csv", sep = ""),
col_types = cols(
diff --git a/src/main/java/org/torproject/metrics/stats/onionperf/Main.java
b/src/main/java/org/torproject/metrics/stats/onionperf/Main.java
index a75cd1b..8fb762d 100644
--- a/src/main/java/org/torproject/metrics/stats/onionperf/Main.java
+++ b/src/main/java/org/torproject/metrics/stats/onionperf/Main.java
@@ -55,6 +55,9 @@ public class Main {
queryBuildTimes(connection));
writeStatistics(new File(baseDir, "stats/latencies.csv").toPath(),
queryLatencies(connection));
+writeStatistics(
+new File(baseDir, "stats/onionperf-throughput.csv").toPath(),
+queryThroughput(connection));
disconnectFromDatabase(connection);
log.info("Terminated onionperf module.");
}
@@ -321,6 +324,33 @@ public class Main {
return statistics;
}
+ static List queryThroughput(Connection connection)
+ throws SQLException {
+log.info("Querying throughput statistics from database.");
+Li
[tor-commits] [metrics-web/release] Include Torperf data in performance graphs again.
commit 896935d33cda602d9b5656a9deed392889b50ffc
Author: Karsten Loesing
Date: Wed Jun 19 04:26:35 2019 -0700
Include Torperf data in performance graphs again.
Fixes #30919.
---
src/main/sql/onionperf/init-onionperf.sql | 8
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/src/main/sql/onionperf/init-onionperf.sql
b/src/main/sql/onionperf/init-onionperf.sql
index b7c41d6..01bfef1 100644
--- a/src/main/sql/onionperf/init-onionperf.sql
+++ b/src/main/sql/onionperf/init-onionperf.sql
@@ -80,7 +80,7 @@ SELECT DATE(start) AS date,
COUNT(*) AS requests
FROM measurements
WHERE DATE(start) < current_date - 1
-AND endpointremote NOT SIMILAR TO '_{56}.onion%'
+AND (endpointremote IS NULL OR endpointremote NOT SIMILAR TO '_{56}.onion%')
GROUP BY date, filesize, source, server
UNION
SELECT DATE(start) AS date,
@@ -98,7 +98,7 @@ SELECT DATE(start) AS date,
COUNT(*) AS requests
FROM measurements
WHERE DATE(start) < current_date - 1
-AND endpointremote NOT SIMILAR TO '_{56}.onion%'
+AND (endpointremote IS NULL OR endpointremote NOT SIMILAR TO '_{56}.onion%')
GROUP BY date, filesize, 3, server) sub
ORDER BY date, filesize, source, server;
@@ -140,7 +140,7 @@ WITH filtered_measurements AS (
WHERE DATE(start) < current_date - 1
AND datarequest > 0
AND dataresponse > 0
- AND endpointremote NOT SIMILAR TO '_{56}.onion%'
+ AND (endpointremote IS NULL OR endpointremote NOT SIMILAR TO '_{56}.onion%')
), quartiles AS (
SELECT date,
source,
@@ -185,7 +185,7 @@ WITH filtered_measurements AS (
ELSE NULL END AS kbps
FROM measurements
WHERE DATE(start) < current_date - 1
- AND endpointremote NOT SIMILAR TO '_{56}.onion%'
+ AND (endpointremote IS NULL OR endpointremote NOT SIMILAR TO '_{56}.onion%')
), quartiles AS (
SELECT date,
source,
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Fix unknown issue with spread() function.
commit b3e80fce17f04cf7c83ef3843d2de0cc3bfef389
Author: Karsten Loesing
Date: Fri May 3 09:28:06 2019 +0200
Fix unknown issue with spread() function.
This is a hotfix to work around the issue described in #30351.
Hopefully, we'll come up with a better fix that doesn't go backwards
from tidyr to reshape2.
---
src/main/R/rserver/rserve-init.R | 21 -
1 file changed, 12 insertions(+), 9 deletions(-)
diff --git a/src/main/R/rserver/rserve-init.R b/src/main/R/rserver/rserve-init.R
index 5a47550..3d6dad7 100644
--- a/src/main/R/rserver/rserve-init.R
+++ b/src/main/R/rserver/rserve-init.R
@@ -1,3 +1,4 @@
+require(reshape2)
require(ggplot2)
require(RColorBrewer)
require(scales)
@@ -436,7 +437,7 @@ prepare_platforms <- function(start_p = NULL, end_p = NULL)
{
filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
mutate(platform = tolower(platform)) %>%
-spread(platform, relays)
+dcast(date ~ platform, value.var = "relays")
}
plot_platforms <- function(start_p, end_p, path_p) {
@@ -700,7 +701,7 @@ prepare_connbidirect <- function(start_p = NULL, end_p =
NULL) {
filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
mutate(quantile = paste("X", quantile, sep = ""),
fraction = fraction / 100) %>%
-spread(quantile, fraction) %>%
+dcast(date + direction ~ quantile, value.var = "fraction") %>%
rename(q1 = X0.25, md = X0.5, q3 = X0.75)
}
@@ -752,7 +753,8 @@ prepare_bandwidth_flags <- function(start_p = NULL, end_p =
NULL) {
filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
filter(!is.na(have_exit_flag)) %>%
filter(!is.na(have_guard_flag)) %>%
-spread(variable, value)
+dcast(date + have_guard_flag + have_exit_flag ~ variable,
+ value.var = "value")
}
plot_bandwidth_flags <- function(start_p, end_p, path_p) {
@@ -1058,7 +1060,7 @@ prepare_advbwdist_perc <- function(start_p = NULL, end_p
= NULL, p_p = NULL) {
transmute(date, percentile = as.factor(percentile),
variable = ifelse(is.na(isexit), "all", "exits"),
advbw = advbw * 8 / 1e9) %>%
-spread(variable, advbw) %>%
+dcast(date + percentile ~ variable, value.var = "advbw") %>%
rename(p = percentile)
}
@@ -1096,7 +1098,7 @@ prepare_advbwdist_relay <- function(start_p = NULL, end_p
= NULL, n_p = NULL) {
transmute(date, relay = as.factor(relay),
variable = ifelse(is.na(isexit), "all", "exits"),
advbw = advbw * 8 / 1e9) %>%
-spread(variable, advbw) %>%
+dcast(date + relay ~ variable, value.var = "advbw") %>%
rename(n = relay)
}
@@ -1194,7 +1196,7 @@ prepare_webstats_tb <- function(start_p = NULL, end_p =
NULL) {
filter(request_type %in% c("tbid", "tbsd", "tbup", "tbur")) %>%
group_by(log_date, request_type) %>%
summarize(count = sum(count)) %>%
-spread(request_type, count) %>%
+dcast(log_date ~ request_type, value.var = "count") %>%
rename(date = log_date, initial_downloads = tbid,
signature_downloads = tbsd, update_pings = tbup,
update_requests = tbur)
@@ -1239,7 +1241,7 @@ prepare_webstats_tb_platform <- function(start_p = NULL,
end_p = NULL) {
filter(request_type %in% c("tbid", "tbup")) %>%
group_by(log_date, platform, request_type) %>%
summarize(count = sum(count)) %>%
-spread(request_type, count, fill = 0) %>%
+dcast(log_date + platform ~ request_type, value.var = "count") %>%
rename(date = log_date, initial_downloads = tbid, update_pings = tbup)
}
@@ -1287,7 +1289,7 @@ prepare_webstats_tb_locale <- function(start_p = NULL,
end_p = NULL) {
group_by(date, locale, request_type) %>%
summarize(count = sum(count)) %>%
mutate(request_type = factor(request_type, levels = c("tbid", "tbup"))) %>%
-spread(request_type, count, fill = 0) %>%
+dcast(date + locale ~ request_type, value.var = "count") %>%
rename(initial_downloads = tbid, update_pings = tbup)
}
@@ -1342,7 +1344,8 @@ prepare_webstats_tm <- function(start_p = NULL, end_p =
NULL) {
group_by(log_date, request_type) %>%
summarize(count = sum(count)) %>%
mutate(request_type = factor(request_type, levels = c("tmid", "tmup"))) %>%
-spread(request_type, count, drop = FALSE, fill = 0) %>%
+dcast(log_date ~ request_type, value.var = "count", drop = FALSE,
+ fill = 0) %>%
rename(date = log_date, initial_downloads = tmid, update_pings = tmup)
}
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Fix two issues found using metrics-test.
commit 5d98d4806e01169f226bace3fc095298844d4427
Author: Karsten Loesing
Date: Mon Sep 16 16:29:10 2019 +0200
Fix two issues found using metrics-test.
- Make unit test locale-independent, similar to Onionoo's 860228c.
- Replace deprecated method Class.newInstance().
---
.../java/org/torproject/metrics/stats/hidserv/Aggregator.java | 5 +++--
.../metrics/stats/hidserv/ComputedNetworkFractions.java| 5 +++--
.../org/torproject/metrics/stats/hidserv/DocumentStore.java| 6 --
.../metrics/stats/hidserv/ExtrapolatedHidServStats.java| 10 ++
.../java/org/torproject/metrics/stats/hidserv/Simulate.java| 9 +
5 files changed, 21 insertions(+), 14 deletions(-)
diff --git a/src/main/java/org/torproject/metrics/stats/hidserv/Aggregator.java
b/src/main/java/org/torproject/metrics/stats/hidserv/Aggregator.java
index 36e7967..12a89a3 100644
--- a/src/main/java/org/torproject/metrics/stats/hidserv/Aggregator.java
+++ b/src/main/java/org/torproject/metrics/stats/hidserv/Aggregator.java
@@ -13,6 +13,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
@@ -176,8 +177,8 @@ public class Aggregator {
/* Put together all aggregated values in a single line. */
String date = e.getKey();
int numStats = weightedValues.size();
-sb.append(String.format("%s,%s,%.0f,%.0f,%.0f,%.8f,%d%n", date,
-type, weightedMean, weightedMedian, weightedInterquartileMean,
+sb.append(String.format(Locale.US, "%s,%s,%.0f,%.0f,%.0f,%.8f,%d%n",
+date, type, weightedMean, weightedMedian,
weightedInterquartileMean,
sumFraction, numStats));
}
}
diff --git
a/src/main/java/org/torproject/metrics/stats/hidserv/ComputedNetworkFractions.java
b/src/main/java/org/torproject/metrics/stats/hidserv/ComputedNetworkFractions.java
index d5fd279..835ac6f 100644
---
a/src/main/java/org/torproject/metrics/stats/hidserv/ComputedNetworkFractions.java
+++
b/src/main/java/org/torproject/metrics/stats/hidserv/ComputedNetworkFractions.java
@@ -8,6 +8,7 @@ import org.slf4j.LoggerFactory;
import java.util.Collections;
import java.util.HashMap;
+import java.util.Locale;
import java.util.Map;
/** Computed fraction of hidden-service activity that a single relay is
@@ -116,9 +117,9 @@ public class ComputedNetworkFractions implements Document {
validAfterDate);
String second = validAfterHour
+ (this.fractionRendRelayedCells == 0.0 ? ","
-: String.format(",%f", this.fractionRendRelayedCells))
+: String.format(Locale.US, ",%f", this.fractionRendRelayedCells))
+ (this.fractionDirOnionsSeen == 0.0 ? ","
-: String.format(",%f", this.fractionDirOnionsSeen));
+: String.format(Locale.US, ",%f", this.fractionDirOnionsSeen));
return new String[] { first, second };
}
diff --git
a/src/main/java/org/torproject/metrics/stats/hidserv/DocumentStore.java
b/src/main/java/org/torproject/metrics/stats/hidserv/DocumentStore.java
index f3163e1..9061aa3 100644
--- a/src/main/java/org/torproject/metrics/stats/hidserv/DocumentStore.java
+++ b/src/main/java/org/torproject/metrics/stats/hidserv/DocumentStore.java
@@ -13,6 +13,7 @@ import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.LineNumberReader;
+import java.lang.reflect.InvocationTargetException;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
@@ -136,7 +137,7 @@ public class DocumentStore {
&& !formattedString0.startsWith(prefix)) {
/* Skip line not starting with prefix. */
} else {
- T document = this.clazz.newInstance();
+ T document = this.clazz.getDeclaredConstructor().newInstance();
if (!document.parse(new String[] { formattedString0,
line.substring(1) })) {
log.warn("Unable to read line {} from {}. Not retrieving any "
@@ -151,7 +152,8 @@ public class DocumentStore {
log.warn("Unable to read {}. Not retrieving any previously stored "
+ "documents.", documentFile.getAbsolutePath(), e);
return null;
-} catch (InstantiationException | IllegalAccessException e) {
+} catch (InstantiationException | IllegalAccessException
+| NoSuchMethodException | InvocationTargetException e) {
log.warn("Unable to read {}. Cannot instantiate document object.",
documentFile.getAbsolutePath(), e);
return null;
diff --git
a/src/main/java/org/torproject/metrics/stats/hidserv/ExtrapolatedHidServStats.java
b/src/main/java/org/torproject/metrics/stats/hidserv/ExtrapolatedHidServStats.java
index
[tor-commits] [metrics-web/release] Re-add the total relay bandwidth graph.
commit c71f0325e1f666ebf7a2fcc73724c2747b290690
Author: Karsten Loesing
Date: Mon Aug 5 10:48:41 2019 +0200
Re-add the total relay bandwidth graph.
Implements 30883.
---
src/main/R/rserver/rserve-init.R | 24 ++
src/main/resources/web.xml | 4
src/main/resources/web/json/categories.json| 1 +
src/main/resources/web/json/metrics.json | 11 ++
.../resources/web/jsps/reproducible-metrics.jsp| 12 ++-
src/main/resources/web/jsps/stats.jsp | 21 +++
6 files changed, 68 insertions(+), 5 deletions(-)
diff --git a/src/main/R/rserver/rserve-init.R b/src/main/R/rserver/rserve-init.R
index f74fd03..af74ef7 100644
--- a/src/main/R/rserver/rserve-init.R
+++ b/src/main/R/rserver/rserve-init.R
@@ -829,6 +829,30 @@ plot_bandwidth_flags <- function(start_p, end_p, path_p) {
ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}
+prepare_bandwidth <- function(start_p = NULL, end_p = NULL) {
+ prepare_bandwidth_flags(start_p, end_p) %>%
+group_by(date) %>%
+summarize(advbw = sum(advbw), bwhist = sum(bwhist))
+}
+
+plot_bandwidth <- function(start_p, end_p, path_p) {
+ prepare_bandwidth(start_p, end_p) %>%
+gather(variable, value, -date) %>%
+ggplot(aes(x = date, y = value, colour = variable)) +
+geom_line() +
+scale_x_date(name = "", breaks = custom_breaks,
+ labels = custom_labels, minor_breaks = custom_minor_breaks) +
+scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"),
+ limits = c(0, NA)) +
+scale_colour_hue(name = "", h.start = 90,
+breaks = c("advbw", "bwhist"),
+labels = c("Advertised bandwidth", "Bandwidth history")) +
+ggtitle("Total relay bandwidth") +
+labs(caption = copyright_notice) +
+theme(legend.position = "top")
+ ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
+}
+
prepare_userstats_relay_country <- function(start_p = NULL, end_p = NULL,
country_p = NULL, events_p = NULL) {
read_csv(file = paste(stats_dir, "clients.csv", sep = ""),
diff --git a/src/main/resources/web.xml b/src/main/resources/web.xml
index f92813a..2c68fd7 100644
--- a/src/main/resources/web.xml
+++ b/src/main/resources/web.xml
@@ -30,6 +30,7 @@
/relayflags.html
/versions.html
/platforms.html
+/bandwidth.html
/bandwidth-flags.html
/dirbytes.html
/advbwdist-perc.html
@@ -118,6 +119,9 @@
/platforms.png
/platforms.pdf
/platforms.csv
+/bandwidth.png
+/bandwidth.pdf
+/bandwidth.csv
/bandwidth-flags.png
/bandwidth-flags.pdf
/bandwidth-flags.csv
diff --git a/src/main/resources/web/json/categories.json
b/src/main/resources/web/json/categories.json
index ad0df11..89742bc 100644
--- a/src/main/resources/web/json/categories.json
+++ b/src/main/resources/web/json/categories.json
@@ -43,6 +43,7 @@
"summary": "How much traffic the Tor network can handle and how much
traffic there is.",
"description": "We measure total available bandwidth and current capacity
by aggregating what relays and bridges report to directory authorities.",
"metrics": [
+ "bandwidth",
"bandwidth-flags",
"advbw-ipv6",
"advbwdist-perc",
diff --git a/src/main/resources/web/json/metrics.json
b/src/main/resources/web/json/metrics.json
index 08e8c19..e6bab04 100644
--- a/src/main/resources/web/json/metrics.json
+++ b/src/main/resources/web/json/metrics.json
@@ -67,6 +67,17 @@
]
},
{
+"id": "bandwidth",
+"title": "Total relay bandwidth",
+"type": "Graph",
+"description": "This graph shows the total advertised and consumed bandwidth of all relays in the network.",
+"function": "bandwidth",
+"parameters": [
+ "start",
+ "end"
+]
+ },
+ {
"id": "bandwidth-flags",
"title": "Advertised and consumed bandwidth by relay flags",
"type": "Graph",
diff --git a/src/main/resources/web/jsps/reproducible-metrics.jsp
b/src/main/resources/web/jsps/reproducible-metrics.jsp
index aee4d5c..d6e7d0e 100644
--- a/src/main/resources/web/jsps/reproducible-metrics.jsp
+++ b/src/main/resources/web/jsps/reproducible-metrics.jsp
@@ -431,6 +431,7 @@ Relays self-report their advertised bandwidth in their
server descriptors which
The following description applies to the following graphs:
+Total relay bandwidth (just the advertised bandwidth part; for the
consumed bandwidth part see below) graph
Advertised and consumed bandwidth
[tor-commits] [metrics-web/release] Adds GitLab CI for tests and checks
commit d4086dde65cb51ca1a39340b1b7a4e9d54981914 Author: Iain R. Learmonth Date: Thu Sep 5 13:44:34 2019 +0100 Adds GitLab CI for tests and checks Configuration is held in the .gitlab-ci.yml file. Fixes: #31402 --- .gitlab-ci.yml | 28 1 file changed, 28 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000..d10fcb8 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,28 @@ +variables: + GIT_STRATEGY: clone + METRICS_LIB_VERSION: "2.6.2" + EXONERATOR_VERSION: "4.1.0" + JAVA_TOOL_OPTIONS: "-Dfile.encoding=UTF-8" + +stages: + - test + +test: + stage: test + image: debian:buster + script: + - apt update + - apt -y install default-jdk ant ivy git curl + - git submodule init + - git submodule update + - mkdir lib + - mkdir tmp + - pushd tmp + - curl https://dist.torproject.org/metrics-lib/$METRICS_LIB_VERSION/metrics-lib-$METRICS_LIB_VERSION.tar.gz | tar xzf - + - curl https://dist.torproject.org/exonerator/$EXONERATOR_VERSION/exonerator-$EXONERATOR_VERSION.tar.gz | tar xzf - + - popd + - mv tmp/metrics-lib-$METRICS_LIB_VERSION/generated/dist/metrics-lib-$METRICS_LIB_VERSION-thin.jar lib + - mv tmp/exonerator-$EXONERATOR_VERSION/generated/dist/exonerator-$EXONERATOR_VERSION-thin.jar lib + - ant -lib /usr/share/java resolve + - ant test + - ant checks ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Update CollecTor page to include bandwidth files.
commit d31b049cb1ae146a5bb4e418dcda91aabf0275d4 Author: Karsten Loesing Date: Wed May 15 16:53:29 2019 +0200 Update CollecTor page to include bandwidth files. Fixes #30507. --- src/main/resources/web/jsps/collector.jsp | 12 +++- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/resources/web/jsps/collector.jsp b/src/main/resources/web/jsps/collector.jsp index 1adc9b3..a05b7e0 100644 --- a/src/main/resources/web/jsps/collector.jsp +++ b/src/main/resources/web/jsps/collector.jsp @@ -192,8 +192,9 @@ Bandwidth Files - @type bandwidth-file 1.2 - + @type bandwidth-file 1.0 + recent + archive @@ -778,15 +779,16 @@ detail on a separate page. Bandwidth Files -@type bandwidth-file 1.2 +@type bandwidth-file 1.0 + recent + archive # Bandwidth authority metrics as defined in the https://gitweb.torproject.org/torspec.git/tree/bandwidth-file-spec.txt;>bandwidth-file-spec. These are available from a DirPort's -/tor/status-vote/next/bandwidth url and CollecTor (both are -https://trac.torproject.org/projects/tor/ticket/21377;>pending). +/tor/status-vote/next/bandwidth url and CollecTor. ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Tweak documentation of new(ly updated) graphs.
commit 3f252fce7f4c3bcfc1a1f53ca0c8f2ff9df31659 Author: Karsten Loesing Date: Tue Jun 11 16:36:06 2019 +0200 Tweak documentation of new(ly updated) graphs. --- src/main/resources/web/jsps/stats.jsp | 10 ++ 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/resources/web/jsps/stats.jsp b/src/main/resources/web/jsps/stats.jsp index 68c4114..5d653cb 100644 --- a/src/main/resources/web/jsps/stats.jsp +++ b/src/main/resources/web/jsps/stats.jsp @@ -50,6 +50,8 @@ https://metrics.torproject.org/identifier.csv October 28, 2018: Added and/or removed columns to Tor Browser downloads and updates by platform and Tor Browser downloads and updates by locale graphs. December 20, 2018: Removed source parameters and output rows with aggregates over all sources from Time to download files over Tor, Timeouts and failures of downloading files over Tor, Circuit build times, Circuit round-trip latencies graphs. December 20, 2018: Removed two graphs Total relay bandwidth and Consumed bandwidth by Exit/Guard flag combination, and updated the data format of the Advertised and consumed bandwidth by relay flags graph to cover all data previously contained in the first two graphs. +May 29, 2019: Extended Circuit round-trip latencies graph to contain high/low values. +June 2, 2019: Added Throughput graph. @@ -563,11 +565,11 @@ Performance # date: UTC date (-MM-DD) when download performance was measured. source: Name of the OnionPerf or Torperf service performing measurements. server: Either "public" if the request was made to a server on the public internet, or "onion" if the request was made to a version 2 onion server. -low: Lowest latency within 1.5 IQR of lower quartile (lower whisker in a boxplot) of time in milliseconds between sending the HTTP request and receiving the HTTP response header. +low: Lowest time in milliseconds between sending the HTTP request and receiving the HTTP response header within 1.5 IQR of lower quartile (lower whisker in a boxplot). q1: First quartile of time in milliseconds between sending the HTTP request and receiving the HTTP response header. md: Median of time in milliseconds between sending the HTTP request and receiving the HTTP response header. q3: Third quartile of time in milliseconds between sending the HTTP request and receiving the HTTP response header. -high: Highest latency within 1.5 IQR of upper quartile (upper whisker in a boxplot) of time in milliseconds between sending the HTTP request and receiving the HTTP response header. +high: Highest time in milliseconds between sending the HTTP request and receiving the HTTP response header within 1.5 IQR of upper quartile (upper whisker in a boxplot). Throughput @@ -589,11 +591,11 @@ Performance # date: UTC date (-MM-DD) when download performance was measured. source: Name of the OnionPerf or Torperf service performing measurements. server: Either "public" if the request was made to a server on the public internet, or "onion" if the request was made to a version 2 onion server. -low: Lowest measured throughput within 1.5 IQR of lower quartile (lower whisker in a boxplot) in kilobits per second. +low: Lowest measured throughput in kilobits per second within 1.5 IQR of lower quartile (lower whisker in a boxplot). q1: First quartile of measured throughput in kilobits per second. md: Median of measured throughput in kilobits per second. q3: Third quartile of measured throughput in kilobits per second. -high: Highest measured throughput within 1.5 IQR of upper quartile (upper whisker in a boxplot) in kilobits per second. +high: Highest measured throughput in kilobits per second within 1.5 IQR of upper quartile (upper whisker in a boxplot). ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Fix links on CollecTor page, again.
commit c3d9c42d0743e9a15c117ab778802fbf7afc3ff1 Author: Karsten Loesing Date: Wed Aug 21 08:33:05 2019 +0200 Fix links on CollecTor page, again. Include metrics-lib-*-thin.jar rather than metrics-lib-*.jar. Re-closes #24792. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index d8c0c07..038a82f 100644 --- a/build.xml +++ b/build.xml @@ -83,7 +83,7 @@ - + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Ignore v3 onion OnionPerf measurements.
commit 7af8f108136b5bb37a0e88f794ce7154c4660ba9
Author: Karsten Loesing
Date: Fri Mar 29 15:14:10 2019 +0100
Ignore v3 onion OnionPerf measurements.
We're later going to include these measurements in graphs, but that
requires more changes. For now, let's ignore these measurements rather
than include them in the v2 onion results. They will be in the
database, so we'll be able to make those graphs including past
measurements.
---
src/main/sql/onionperf/init-onionperf.sql | 6 +-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/src/main/sql/onionperf/init-onionperf.sql
b/src/main/sql/onionperf/init-onionperf.sql
index e9af3b1..7c0176e 100644
--- a/src/main/sql/onionperf/init-onionperf.sql
+++ b/src/main/sql/onionperf/init-onionperf.sql
@@ -36,7 +36,7 @@ CREATE TABLE IF NOT EXISTS measurements (
used_by INTEGER,
endpointlocal CHARACTER VARYING(64),
endpointproxy CHARACTER VARYING(64),
- endpointremote CHARACTER VARYING(64),
+ endpointremote CHARACTER VARYING(96),
hostnamelocal CHARACTER VARYING(64),
hostnameremote CHARACTER VARYING(64),
sourceaddress CHARACTER VARYING(64),
@@ -80,6 +80,7 @@ SELECT DATE(start) AS date,
COUNT(*) AS requests
FROM measurements
WHERE DATE(start) < current_date - 1
+AND endpointremote NOT SIMILAR TO '_{56}.onion%'
GROUP BY date, filesize, source, server
UNION
SELECT DATE(start) AS date,
@@ -97,6 +98,7 @@ SELECT DATE(start) AS date,
COUNT(*) AS requests
FROM measurements
WHERE DATE(start) < current_date - 1
+AND endpointremote NOT SIMILAR TO '_{56}.onion%'
GROUP BY date, filesize, 3, server) sub
ORDER BY date, filesize, source, server;
@@ -145,6 +147,7 @@ FROM measurements
WHERE DATE(start) < current_date - 1
AND datarequest > 0
AND dataresponse > 0
+AND endpointremote NOT SIMILAR TO '_{56}.onion%'
GROUP BY date, source, server
UNION
SELECT DATE(start) AS date,
@@ -157,6 +160,7 @@ FROM measurements
WHERE DATE(start) < current_date - 1
AND datarequest > 0
AND dataresponse > 0
+AND endpointremote NOT SIMILAR TO '_{56}.onion%'
GROUP BY date, 2, server) sub
ORDER BY date, source, server;
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Stop signing jars.
commit 46059cf2a7bd58c651f44b89aaa009c911fe3084 Author: Karsten Loesing Date: Wed Nov 28 10:14:21 2018 +0100 Stop signing jars. Implements #28584. --- CERT| 21 - src/build | 2 +- src/submods/metrics-lib | 2 +- 3 files changed, 2 insertions(+), 23 deletions(-) diff --git a/CERT b/CERT deleted file mode 100644 index b90b397..000 --- a/CERT +++ /dev/null @@ -1,21 +0,0 @@ --BEGIN CERTIFICATE- -MIIDaTCCAlGgAwIBAgIELle0dTANBgkqhkiG9w0BAQsFADBlMQswCQYDVQQGEwJV -UzELMAkGA1UECBMCV0ExEDAOBgNVBAcTB1NlYXR0bGUxHTAbBgNVBAoTFFRoZSBU -b3IgUHJvamVjdCwgSW5jMRgwFgYDVQQDEw9LYXJzdGVuIExvZXNpbmcwHhcNMTgw -ODI4MDcwNjM2WhcNMTgxMTI2MDcwNjM2WjBlMQswCQYDVQQGEwJVUzELMAkGA1UE -CBMCV0ExEDAOBgNVBAcTB1NlYXR0bGUxHTAbBgNVBAoTFFRoZSBUb3IgUHJvamVj -dCwgSW5jMRgwFgYDVQQDEw9LYXJzdGVuIExvZXNpbmcwggEiMA0GCSqGSIb3DQEB -AQUAA4IBDwAwggEKAoIBAQChXn+IUp+o6G+k4ffxk3TkxZb3iXfiG7byNsG63olU -6aTpAjDMeaT4ctUwxH4+56Sbcf/wB0vEFBbX8MyRd1eY02PKwMVJ6VBhjOQcIlrd -Qw+VAhKTcEIv4yiR0BWapQyR07pgmKirYVjN6s6ef8NJzUptpxLlaYJ3ZfQfc4aE -MXzScgaccwDFIWQ661lzLGCfeSxxa3Xy4wWsGwzNzLITYrrABcbg7yogLo2btNvD -oEwGL3/baQdhl0dra6biVCZr9ydn3Hg57S55pUU0rBY25id78zUO8xrfNHw54wwX -lOblGt75OOkahP/ZZSBxxoiknJ6y5VQV8y+noA4vigXFAgMBAAGjITAfMB0GA1Ud -DgQWBBSeh60M+/wMYyYhlxtuff2Hk9n7bzANBgkqhkiG9w0BAQsFAAOCAQEAkXZs -3T3GTkZ+EGvZG5puzKdgZiSsLgIy25xdWsIx147AIZEJFKjEAtbu0osMpkTa96B6 -a+BHf7PTjQUuH3YOEmeW9ab8pwu5SRijCq2qkuvjjSLBcJzWnalcKDYYvoQte1// -Di8JqpRXCw20WY2bldTiafyG80E0RGfiX2I8vbDiPIhjwz9Wox8Q1rw1c9T/vRn9 -pI8FrHgTnDO6R54yD25QSpsj+hC+IDkFKO17vGCIaJrPG5o6th438ijEwJsG+LRB -4zKKKsFTby7UJI3Ag8xolIhsBkRZO2j4Na35i15SZ7QJNj9J5g171z8RyOmyIQbg -q7OXN2iiRIxiIJwoQw== --END CERTIFICATE- diff --git a/src/build b/src/build index 08514a3..e639c69 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 08514a32afefbeef848b80f9a338ee840c282604 +Subproject commit e639c697e9e94c6dbb26e946e5247c20a62c0661 diff --git a/src/submods/metrics-lib b/src/submods/metrics-lib index 603a439..23927c2 16 --- a/src/submods/metrics-lib +++ b/src/submods/metrics-lib @@ -1 +1 @@ -Subproject commit 603a439f802c6d4a8b29367ce13b345ae8cf02bc +Subproject commit 23927c2777f273c42ad3e75fc0a2940ed8eb4bf6 ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Updates fallback directories
commit b6223ecf7a7242e8e4c47a0c5fb70f5ffa35c845 Author: Iain R. Learmonth Date: Wed Mar 27 19:18:28 2019 + Updates fallback directories --- src/main/resources/web/js/rs/fallback_dir.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/web/js/rs/fallback_dir.js b/src/main/resources/web/js/rs/fallback_dir.js index 93f97c5..f1a000f 100644 --- a/src/main/resources/web/js/rs/fallback_dir.js +++ b/src/main/resources/web/js/rs/fallback_dir.js @@ -8,7 +8,7 @@ To update run: python3 scripts/fallback_dir.py > js/fallback_dir.js */ -var fallbackDirs = ["001524DD403D729F08F7E5D77813EF12756CFA8D", "0111BA9B604669E636FFD5B503F382A4B7AD6E80", "025B66CEBC070FCB0519D206CF0CF4965C20C96E", "0756B7CD4DFC8182BE23143FAC0642F515182CEB", "0B85617241252517E8ECF2CFC7F4C1A32DCD153F", "0BEA4A88D069753218EAAAD6D22EA87B9A1319D6", "0CF8F3E6590F45D50B70F2F7DA6605ECA6CD408F", "0D3EBA17E1C78F1E9900BABDB23861D46FCAF163", "0E8C0C8315B66DB5F703804B3889A1DD66C67CE0", "11DF0017A43AF1F08825CD5D973297F81AB00FF3", "12AD30E5D25AA67F519780E2111E611A455FDC89", "12FD624EE73CEF37137C90D38B2406A66F68FAA2", "136F9299A5009A4E0E96494E723BDB556FB0A26B", "16102E458460349EE45C0901DAA6C30094A9BBEA", "175921396C7C426309AB03775A9930B6F611F794", "185663B7C12777F052B2C2D23D7A239D8DA88A0F", "1938EBACBB1A7BFA888D9623C90061130E63BB3F", "1AE039EE0B11DB79E4B4B29CBA9F752864A0259E", "1C90D3AEADFF3BCD079810632C8B85637924A58E", "1DBAED235E3957DE1ABD25B4206BE71406FB61F8", "1ECD73B936CB6E6B3CD647CC204F108D9DF2C9F7", "1F6ABD086F40B890A33C93CC4606EE68B31C9556", "1FA8F638 298645BE58AC905276680889CB795A94", "20462CBA5DA4C2D963567D17D0B7249718114A68", "204DFD2A2C6A0DC1FA0EACB495218E0B661704FD", "230A8B2A8BA861210D9B4BA97745AEC217A94207", "231C2B9C8C31C295C472D031E06964834B745996", "2BA2C8E96B2590E1072AECE2BDB5C48921BF8510", "2CDCFED0142B28B002E89D305CBA2E26063FADE2", "2F0F32AB1E5B943CA7D062C03F18960C86E70D94", "30C19B81981F450C402306E2E7CFB6C3F79CB6B2", "328E54981C6DDD7D89B89E418724A4A7881E3192", "330CD3DB6AD266DC70CDB512B036957D03D9BC59", "33DA0CAB7C27812EFF2E22C9705630A54D101FEB", "3711E80B5B04494C971FB0459D4209AB7F2EA799", "379FB450010D17078B3766C2273303C358C3A442", "387B065A38E4DAA16D9D41C2964ECBC4B31D30FF", "39F096961ED2576975C866D450373A9913AFDC92", "3B33F6FCA645AD4E91428A3AF7DC736AD9FB727B", "3C79699D4FBC37DE1A212D5033B56DAE079AC0EF", "3D7E274A87D9A89AF064C13D1EE4CA1F184F2600", "3E53D3979DB07EFD736661C934A1DED14127B684", "4061C553CA88021B8302F0814365070AAE617270", "40E7D6CE5085E4CDDA31D51A29D1457EB53F12AD", "41C59606AFE1D1AA6EC6EF6719690B856F0B6 587", "439D0447772CB107B886F7782DBC201FA26B92D1", "4623A9EC53BFD83155929E56D6F7B55B5E718C24", "46791D156C9B6C255C2665D4D8393EC7DBAA7798", "484A10BA2B8D48A5F0216674C8DD50EF27BC32F3", "489D94333DF66D57FFE34D9D59CC2D97E2CB0053", "4CC9CC9195EC38645B699A33307058624F660CCF", "4F0DB7E687FC7C0AE55C8F243DA8B0EB27FBF1F2", "50586E25BE067FD1F739998550EDDCB1A14CA5B2", "51E1CF613FD6F9F11FE24743C91D6F9981807D82", "52BFADA8BEAA01BA46C8F767F83C18E2FE50C1B9", "587E0A9552E4274B251F29B5B2673D38442EE4BF", "58ED9C9C35E433EE58764D62892B4FFD518A3CD0", "5E56738E7F97AA81DEEF59AF28494293DFBFCCDF", "5F4CD12099AF20FAF9ADFDCEC65316A376D0201C", "616081EC829593AF4232550DE6FFAA1D75B37A90", "68F175CCABE727AA2D2309BCD8789499CEE36ED7", "6A7551EEE18F78A9813096E82BF84F740D32B911", "6EF897645B79B6CB35E853B32506375014DE3621", "72B2B12A3F60408BDBC98C6DF53988D3A0B3F0EE", "7600680249A22080ECC6173FBBF64D6FCF330A61", "763C9556602BD6207771A7A3D958091D44C43228", "774555642FDC1E1D4FDF2E0C31B7CA9501C5C9C7", "775B0FAFDE71AADC23FFC8 782B7BEB1D5A92733E", "789EA6C9AE9ADDD8760903171CFA9AC5741B0C70", "78E2BE744A53631B4AAB781468E94C52AB73968B", "79E169B25E4C7CE99584F6ED06F379478F23E2B8", "7A32C9519D80CA458FC8B034A28F5F6815649A98", "7BB70F8585DFC27E75D692970C0EEB0F22983A63", "7BFB908A3AA5B491DA4CA72CCBEE0E1F2A939B55", "7D05A38E39FC5D29AFE6BE487B9B4DC9E635D09E", "7FA8E7E44F1392A4E40FFC3B69DB3B00091B7FD3", "80AAF8D5956A43C197104CEF2550CD42D165C6FB", "8456DFA94161CDD99E480C2A2992C366C6564410", "855BC2DABE24C861CD887DB9B2E950424B49FC34", "8567AD0A6369ED08527A8A8533A5162AC00F7678", "86C281AD135058238D7A337D546C902BE8505DDE", "88487BDD980BF6E72092EE690E8C51C0AA4A538C", "8C00FA7369A7A308F6A137600F0FA07990D9D451", "8D79F73DCD91FC4F5017422FAC70074D6DB8DD81", "9007C1D8E4F03D506A4A011B907A9E8D04E3C605", "91D23D8A539B83D2FB56AA67ECD4D75CC093AC55", "9285B22F7953D7874604EEE2B470609AD81C74E9", "92CFD9565B24646CAC2D172D3DB503D69E777B8A", "92ECC9E0E2AF81BB954719B189AC362E254AD4A5", "9772EFB535397C942C3AB8804FB35CFFAD012438", "998BF3E D7F70E33D1C307247B9626D9E7573C438", "9A0D54D3A6D2E0767596BF1515E6162A75B3293F", "9A68B85A02318F4E7E87F2828039FBD5D75B0142", "9B31F1F1C1554F9FFB3455911F82E818EF7C7883", "9EC5E097663862DF861A18C32B37C5F82284B27D", "9F2856F6D2B89AD4EF6D5723FAB167DB5A53519A", "9F7D6E6420183C2B76D3CE99624EBC98A21A967E",
[tor-commits] [metrics-web/release] Include op-ab in OnionPerf graphs.
commit 82aa20a17cb8ff827873e6d0a776b9e2967197bb Author: Karsten Loesing Date: Thu Jan 24 12:10:03 2019 +0100 Include op-ab in OnionPerf graphs. Turns out that op-ab's domain name matches our '%.onion%' regex, that we're using to distinguish public from onion server requests. Trying a bit harder to distinguish the two. Related to #29107. --- src/main/sql/onionperf/init-onionperf.sql | 8 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/sql/onionperf/init-onionperf.sql b/src/main/sql/onionperf/init-onionperf.sql index 49eff4a..e9af3b1 100644 --- a/src/main/sql/onionperf/init-onionperf.sql +++ b/src/main/sql/onionperf/init-onionperf.sql @@ -68,7 +68,7 @@ FROM ( SELECT DATE(start) AS date, filesize, source, - CASE WHEN endpointremote LIKE '%.onion%' THEN 'onion' + CASE WHEN endpointremote LIKE '%.onion:%' THEN 'onion' ELSE 'public' END AS server, CASE WHEN COUNT(*) > 0 THEN PERCENTILE_CONT(ARRAY[0.25,0.5,0.75]) WITHIN GROUP(ORDER BY datacomplete) @@ -85,7 +85,7 @@ UNION SELECT DATE(start) AS date, filesize, '' AS source, - CASE WHEN endpointremote LIKE '%.onion%' THEN 'onion' + CASE WHEN endpointremote LIKE '%.onion:%' THEN 'onion' ELSE 'public' END AS server, CASE WHEN COUNT(*) > 0 THEN PERCENTILE_CONT(ARRAY[0.25,0.5,0.75]) WITHIN GROUP(ORDER BY datacomplete) @@ -137,7 +137,7 @@ SELECT date, FROM ( SELECT DATE(start) AS date, source, - CASE WHEN endpointremote LIKE '%.onion%' THEN 'onion' + CASE WHEN endpointremote LIKE '%.onion:%' THEN 'onion' ELSE 'public' END AS server, PERCENTILE_CONT(ARRAY[0.25,0.5,0.75]) WITHIN GROUP(ORDER BY dataresponse - datarequest) AS q @@ -149,7 +149,7 @@ GROUP BY date, source, server UNION SELECT DATE(start) AS date, '' AS source, - CASE WHEN endpointremote LIKE '%.onion%' THEN 'onion' + CASE WHEN endpointremote LIKE '%.onion:%' THEN 'onion' ELSE 'public' END AS server, PERCENTILE_CONT(ARRAY[0.25,0.5,0.75]) WITHIN GROUP(ORDER BY dataresponse - datarequest) AS q ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Add levels = NULL to col_factor().
commit a9fac06cc120a13dd35ba77983dc2d54aacc75ed
Author: Karsten Loesing
Date: Fri Jan 11 09:30:53 2019 +0100
Add levels = NULL to col_factor().
This is not required for readr_1.3.0 which runs locally here but
required for readr_1.1.1 which runs on the server.
---
src/main/R/rserver/graphs.R | 24
1 file changed, 12 insertions(+), 12 deletions(-)
diff --git a/src/main/R/rserver/graphs.R b/src/main/R/rserver/graphs.R
index 205afbe..18a9d3e 100644
--- a/src/main/R/rserver/graphs.R
+++ b/src/main/R/rserver/graphs.R
@@ -682,7 +682,7 @@ prepare_connbidirect <- function(start_p = NULL, end_p =
NULL) {
read_csv(file = paste(stats_dir, "connbidirect2.csv", sep = ""),
col_types = cols(
date = col_date(format = ""),
-direction = col_factor(),
+direction = col_factor(levels = NULL),
quantile = col_double(),
fraction = col_double())) %>%
filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
@@ -1109,7 +1109,7 @@ prepare_hidserv_dir_onions_seen <- function(start_p =
NULL, end_p = NULL) {
read_csv(file = paste(stats_dir, "hidserv.csv", sep = ""),
col_types = cols(
date = col_date(format = ""),
-type = col_factor(),
+type = col_factor(levels = NULL),
wmean = col_skip(),
wmedian = col_skip(),
wiqm = col_double(),
@@ -1137,7 +1137,7 @@ prepare_hidserv_rend_relayed_cells <- function(start_p =
NULL, end_p = NULL) {
read_csv(file = paste(stats_dir, "hidserv.csv", sep = ""),
col_types = cols(
date = col_date(format = ""),
-type = col_factor(),
+type = col_factor(levels = NULL),
wmean = col_skip(),
wmedian = col_skip(),
wiqm = col_double(),
@@ -1167,7 +1167,7 @@ prepare_webstats_tb <- function(start_p = NULL, end_p =
NULL) {
read_csv(file = paste(stats_dir, "webstats.csv", sep = ""),
col_types = cols(
log_date = col_date(format = ""),
-request_type = col_factor(),
+request_type = col_factor(levels = NULL),
platform = col_skip(),
channel = col_skip(),
locale = col_skip(),
@@ -1210,8 +1210,8 @@ prepare_webstats_tb_platform <- function(start_p = NULL,
end_p = NULL) {
read_csv(file = paste(stats_dir, "webstats.csv", sep = ""),
col_types = cols(
log_date = col_date(format = ""),
-request_type = col_factor(),
-platform = col_factor(),
+request_type = col_factor(levels = NULL),
+platform = col_factor(levels = NULL),
channel = col_skip(),
locale = col_skip(),
incremental = col_skip(),
@@ -1253,10 +1253,10 @@ prepare_webstats_tb_locale <- function(start_p = NULL,
end_p = NULL) {
read_csv(file = paste(stats_dir, "webstats.csv", sep = ""),
col_types = cols(
log_date = col_date(format = ""),
-request_type = col_factor(),
+request_type = col_factor(levels = NULL),
platform = col_skip(),
channel = col_skip(),
-locale = col_factor(),
+locale = col_factor(levels = NULL),
incremental = col_skip(),
count = col_double())) %>%
filter(if (!is.null(start_p)) log_date >= as.Date(start_p) else TRUE) %>%
@@ -1308,7 +1308,7 @@ prepare_webstats_tm <- function(start_p = NULL, end_p =
NULL) {
read_csv(file = paste(stats_dir, "webstats.csv", sep = ""),
col_types = cols(
log_date = col_date(format = ""),
-request_type = col_factor(),
+request_type = col_factor(levels = NULL),
platform = col_skip(),
channel = col_skip(),
locale = col_skip(),
@@ -1348,7 +1348,7 @@ prepare_relays_ipv6 <- function(start_p = NULL, end_p =
NULL) {
read_csv(file = paste(stats_dir, "ipv6servers.csv", sep = ""),
col_types = cols(
valid_after_date = col_date(format = ""),
-server = col_factor(),
+server = col_factor(levels = NULL),
guard_relay = col_skip(),
exit_relay = col_skip(),
announced_ipv6 = col_logical(),
@@ -1395,7 +1395,7 @@ prepare_bridges_ipv6 <- function(start_p = NULL, end_p =
NULL) {
read_csv(file = paste(stats_dir, "ipv6servers.csv", sep = ""),
col_types = cols(
valid_after_date = col_date(format = ""),
-server = col_factor(),
+server = col_factor(levels = NULL),
guard_relay = col_skip(),
exit_relay = col_skip(),
announced_ipv6 = col_logical(),
@@ -1436,7 +1436,7 @@ prepare_advbw_ipv6 <- function(start_p = NULL, end_p =
NULL) {
read_csv(f
[tor-commits] [metrics-web/release] Simplify Rserve setup.
commit e82de493279a0e74b55e5fd66a4056a1cecf19c5 Author: Karsten Loesing Date: Fri Jan 11 11:39:12 2019 +0100 Simplify Rserve setup. --- src/main/R/rserver/Rserv.conf|2 - src/main/R/rserver/graphs.R | 1539 src/main/R/rserver/rserve-init.R | 1609 +- src/main/R/rserver/tables.R | 58 -- 4 files changed, 1600 insertions(+), 1608 deletions(-) diff --git a/src/main/R/rserver/Rserv.conf b/src/main/R/rserver/Rserv.conf deleted file mode 100644 index 1fb3039..000 --- a/src/main/R/rserver/Rserv.conf +++ /dev/null @@ -1,2 +0,0 @@ -workdir /srv/metrics.torproject.org/metrics/website/rserve/workdir -source rserve-init.R diff --git a/src/main/R/rserver/graphs.R b/src/main/R/rserver/graphs.R deleted file mode 100644 index 0d7a90c..000 --- a/src/main/R/rserver/graphs.R +++ /dev/null @@ -1,1539 +0,0 @@ -countrylist <- list( - "ad" = "Andorra", - "ae" = "the United Arab Emirates", - "af" = "Afghanistan", - "ag" = "Antigua and Barbuda", - "ai" = "Anguilla", - "al" = "Albania", - "am" = "Armenia", - "an" = "the Netherlands Antilles", - "ao" = "Angola", - "aq" = "Antarctica", - "ar" = "Argentina", - "as" = "American Samoa", - "at" = "Austria", - "au" = "Australia", - "aw" = "Aruba", - "ax" = "the Aland Islands", - "az" = "Azerbaijan", - "ba" = "Bosnia and Herzegovina", - "bb" = "Barbados", - "bd" = "Bangladesh", - "be" = "Belgium", - "bf" = "Burkina Faso", - "bg" = "Bulgaria", - "bh" = "Bahrain", - "bi" = "Burundi", - "bj" = "Benin", - "bl" = "Saint Bartelemey", - "bm" = "Bermuda", - "bn" = "Brunei", - "bo" = "Bolivia", - "bq" = "Bonaire, Sint Eustatius and Saba", - "br" = "Brazil", - "bs" = "the Bahamas", - "bt" = "Bhutan", - "bv" = "the Bouvet Island", - "bw" = "Botswana", - "by" = "Belarus", - "bz" = "Belize", - "ca" = "Canada", - "cc" = "the Cocos (Keeling) Islands", - "cd" = "the Democratic Republic of the Congo", - "cf" = "Central African Republic", - "cg" = "Congo", - "ch" = "Switzerland", - "ci" = "Côte d'Ivoire", - "ck" = "the Cook Islands", - "cl" = "Chile", - "cm" = "Cameroon", - "cn" = "China", - "co" = "Colombia", - "cr" = "Costa Rica", - "cu" = "Cuba", - "cv" = "Cape Verde", - "cw" = "Curaçao", - "cx" = "the Christmas Island", - "cy" = "Cyprus", - "cz" = "the Czech Republic", - "de" = "Germany", - "dj" = "Djibouti", - "dk" = "Denmark", - "dm" = "Dominica", - "do" = "the Dominican Republic", - "dz" = "Algeria", - "ec" = "Ecuador", - "ee" = "Estonia", - "eg" = "Egypt", - "eh" = "the Western Sahara", - "er" = "Eritrea", - "es" = "Spain", - "et" = "Ethiopia", - "fi" = "Finland", - "fj" = "Fiji", - "fk" = "the Falkland Islands (Malvinas)", - "fm" = "the Federated States of Micronesia", - "fo" = "the Faroe Islands", - "fr" = "France", - "ga" = "Gabon", - "gb" = "the United Kingdom", - "gd" = "Grenada", - "ge" = "Georgia", - "gf" = "French Guiana", - "gg" = "Guernsey", - "gh" = "Ghana", - "gi" = "Gibraltar", - "gl" = "Greenland", - "gm" = "Gambia", - "gn" = "Guinea", - "gp" = "Guadeloupe", - "gq" = "Equatorial Guinea", - "gr" = "Greece", - "g
[tor-commits] [metrics-web/release] Extend latency graph to contain high/low values.
commit fd251d6c69efeb3d548f915cc40e001e89c1f41f
Author: Karsten Loesing
Date: Wed May 29 16:10:40 2019 +0200
Extend latency graph to contain high/low values.
This patch adds two new lines to the existing circuit round-trip
latencies graph: lowest and highest measurements that are not
outliers.
Implements #29773.
---
src/main/R/rserver/rserve-init.R | 15 --
.../torproject/metrics/stats/onionperf/Main.java | 10 ++--
src/main/resources/web/json/metrics.json | 2 +-
.../resources/web/jsps/reproducible-metrics.jsp| 3 +-
src/main/resources/web/jsps/stats.jsp | 2 +
src/main/sql/onionperf/init-onionperf.sql | 58 +++---
6 files changed, 51 insertions(+), 39 deletions(-)
diff --git a/src/main/R/rserver/rserve-init.R b/src/main/R/rserver/rserve-init.R
index 3d6dad7..88aa5b9 100644
--- a/src/main/R/rserver/rserve-init.R
+++ b/src/main/R/rserver/rserve-init.R
@@ -663,9 +663,11 @@ prepare_onionperf_latencies <- function(start_p = NULL,
end_p = NULL,
date = col_date(format = ""),
source = col_character(),
server = col_character(),
+low = col_double(),
q1 = col_double(),
md = col_double(),
-q3 = col_double())) %>%
+q3 = col_double(),
+high = col_double())) %>%
filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
filter(if (!is.null(server_p)) server == server_p else TRUE)
@@ -675,18 +677,23 @@ plot_onionperf_latencies <- function(start_p, end_p,
server_p, path_p) {
prepare_onionperf_latencies(start_p, end_p, server_p) %>%
filter(source != "") %>%
complete(date = full_seq(date, period = 1), nesting(source)) %>%
-ggplot(aes(x = date, y = md, ymin = q1, ymax = q3, fill = source)) +
+ggplot(aes(x = date, ymin = q1, ymax = q3, fill = source)) +
geom_ribbon(alpha = 0.5) +
-geom_line(aes(colour = source), size = 0.75) +
+geom_line(aes(y = md, colour = source), size = 0.75) +
+geom_line(aes(y = high, colour = source), size = 0.375) +
+geom_line(aes(y = low, colour = source), size = 0.375) +
scale_x_date(name = "", breaks = custom_breaks,
labels = custom_labels, minor_breaks = custom_minor_breaks) +
scale_y_continuous(name = "", labels = unit_format(unit = "ms"),
limits = c(0, NA)) +
scale_fill_hue(name = "Source") +
scale_colour_hue(name = "Source") +
+facet_grid(source ~ ., scales = "free", space = "free") +
ggtitle(paste("Circuit round-trip latencies to", server_p, "server")) +
labs(caption = copyright_notice) +
-theme(legend.position = "top")
+theme(legend.position = "none",
+ strip.text.y = element_text(angle = 0, hjust = 0),
+ strip.background = element_rect(fill = NA))
ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}
diff --git a/src/main/java/org/torproject/metrics/stats/onionperf/Main.java
b/src/main/java/org/torproject/metrics/stats/onionperf/Main.java
index a970434..a75cd1b 100644
--- a/src/main/java/org/torproject/metrics/stats/onionperf/Main.java
+++ b/src/main/java/org/torproject/metrics/stats/onionperf/Main.java
@@ -298,22 +298,24 @@ public class Main {
throws SQLException {
log.info("Querying latency statistics from database.");
List statistics = new ArrayList<>();
-statistics.add("date,source,server,q1,md,q3");
+statistics.add("date,source,server,low,q1,md,q3,high");
Statement st = connection.createStatement();
-String queryString = "SELECT date, source, server, q1, md, q3 "
+String queryString = "SELECT date, source, server, low, q1, md, q3, high "
+ "FROM latencies_stats";
DateFormat dateFormat = new SimpleDateFormat("-MM-dd", Locale.US);
dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
try (ResultSet rs = st.executeQuery(queryString)) {
while (rs.next()) {
-statistics.add(String.format("%s,%s,%s,%d,%d,%d",
+statistics.add(String.format("%s,%s,%s,%d,%d,%d,%d,%d",
dateFormat.format(rs.getDate("date", calendar)),
getStringFromResultSet(rs, "source"),
rs.getString("server"),
+rs.getInt("low"),
rs.getInt("q1"),
rs.getInt("md"),
-rs.getInt("q3")));
+rs.getInt("q3"),
+rs.getInt("high")));
}
}
return statistics;
diff --git a/src/main/r
[tor-commits] [metrics-web/release] Leave gaps for missing data.
commit d1cedb7f2d011f8896f169338db7f0403702ea64
Author: Karsten Loesing
Date: Fri Jan 11 10:48:47 2019 +0100
Leave gaps for missing data.
---
src/main/R/rserver/graphs.R | 47 ++---
1 file changed, 31 insertions(+), 16 deletions(-)
diff --git a/src/main/R/rserver/graphs.R b/src/main/R/rserver/graphs.R
index 18a9d3e..0d7a90c 100644
--- a/src/main/R/rserver/graphs.R
+++ b/src/main/R/rserver/graphs.R
@@ -405,7 +405,9 @@ plot_versions <- function(start_p, end_p, path_p) {
stringsAsFactors = FALSE)
versions <- s[s$version %in% known_versions, ]
visible_versions <- sort(unique(versions$version))
- ggplot(versions, aes(x = date, y = relays, colour = version)) +
+ versions <- versions %>%
+complete(date = full_seq(date, period = 1), nesting(version)) %>%
+ggplot(aes(x = date, y = relays, colour = version)) +
geom_line() +
scale_x_date(name = "", breaks = custom_breaks,
labels = custom_labels, minor_breaks = custom_minor_breaks) +
@@ -433,6 +435,7 @@ prepare_platforms <- function(start_p = NULL, end_p = NULL)
{
plot_platforms <- function(start_p, end_p, path_p) {
prepare_platforms(start_p, end_p) %>%
gather(platform, relays, -date) %>%
+complete(date = full_seq(date, period = 1), nesting(platform)) %>%
ggplot(aes(x = date, y = relays, colour = platform)) +
geom_line() +
scale_x_date(name = "", breaks = custom_breaks,
@@ -470,6 +473,7 @@ prepare_dirbytes <- function(start_p = NULL, end_p = NULL) {
plot_dirbytes <- function(start_p, end_p, path_p) {
prepare_dirbytes(start_p, end_p) %>%
gather(variable, value, -date) %>%
+complete(date = full_seq(date, period = 1), nesting(variable)) %>%
ggplot(aes(x = date, y = value, colour = variable)) +
geom_line() +
scale_x_date(name = "", breaks = custom_breaks,
@@ -695,10 +699,10 @@ prepare_connbidirect <- function(start_p = NULL, end_p =
NULL) {
plot_connbidirect <- function(start_p, end_p, path_p) {
prepare_connbidirect(start_p, end_p) %>%
-ggplot(aes(x = date, y = md, colour = direction)) +
-geom_line(size = 0.75) +
-geom_ribbon(aes(x = date, ymin = q1, ymax = q3,
-fill = direction), alpha = 0.5, show.legend = FALSE) +
+complete(date = full_seq(date, period = 1), nesting(direction)) %>%
+ggplot(aes(x = date, y = md, ymin = q1, ymax = q3, fill = direction)) +
+geom_ribbon(alpha = 0.5) +
+geom_line(aes(colour = direction), size = 0.75) +
scale_x_date(name = "", breaks = custom_breaks,
labels = custom_labels, minor_breaks = custom_minor_breaks) +
scale_y_continuous(name = "", labels = percent, limits = c(0, NA)) +
@@ -1013,11 +1017,12 @@ plot_userstats_bridge_combined <- function(start_p,
end_p, country_p, path_p) {
a <- aggregate(list(mid = (u$high + u$low) / 2),
by = list(transport = u$transport), FUN = sum)
a <- a[order(a$mid, decreasing = TRUE)[1:top], ]
-u <- u[u$transport %in% a$transport, ]
+u <- u[u$transport %in% a$transport, ] %>%
+ complete(date = full_seq(date, period = 1), nesting(country, transport))
title <- paste("Bridge users by transport from ",
countryname(country_p), sep = "")
ggplot(u, aes(x = as.Date(date), ymin = low, ymax = high,
-colour = transport, fill = transport)) +
+ fill = transport)) +
geom_ribbon(alpha = 0.5, size = 0.5) +
scale_x_date(name = "", breaks = custom_breaks,
labels = custom_labels, minor_breaks = custom_minor_breaks) +
@@ -1055,6 +1060,7 @@ plot_advbwdist_perc <- function(start_p, end_p, p_p,
path_p) {
gather(variable, advbw, -c(date, p)) %>%
mutate(variable = ifelse(variable == "all", "All relays",
"Exits only")) %>%
+complete(date = full_seq(date, period = 1), nesting(p, variable)) %>%
ggplot(aes(x = date, y = advbw, colour = p)) +
facet_grid(variable ~ .) +
geom_line() +
@@ -1092,6 +1098,7 @@ plot_advbwdist_relay <- function(start_p, end_p, n_p,
path_p) {
gather(variable, advbw, -c(date, n)) %>%
mutate(variable = ifelse(variable == "all", "All relays",
"Exits only")) %>%
+complete(date = full_seq(date, period = 1), nesting(n, variable)) %>%
ggplot(aes(x = date, y = advbw, colour = n)) +
facet_grid(variable ~ .) +
geom_line() +
@@ -1123,6 +1130,7 @@ prepare_hidserv_dir_onions_seen <- function(start_p =
NULL, end_p = NULL) {
plot_hidserv_dir_onions_seen <- function(start_p, end_p, path_p) {
prepare_hidserv_dir_onions_seen(start_p, end_p) %>%
+complete(date = full_seq(date, period = 1)) %>%
ggplot(aes(x = date, y = onions)) +
geom_line() +
scale_x_date(name
[tor-commits] [metrics-web/release] Re-add no-data-available.png for display in RS.
commit 1d21cebf4b999ced613f444c3278f1b53acf9480 Author: Karsten Loesing Date: Wed Apr 3 09:38:56 2019 +0200 Re-add no-data-available.png for display in RS. Reported on metrics-team@. --- src/main/resources/web/images/no-data-available.png | Bin 0 -> 26209 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/src/main/resources/web/images/no-data-available.png b/src/main/resources/web/images/no-data-available.png new file mode 100644 index 000..36310ad Binary files /dev/null and b/src/main/resources/web/images/no-data-available.png differ ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Remove advbw column from bandwidth.csv.
commit 09cfdfdff4efc1aa1cc60f53f7f1353a6193e6ad
Author: Karsten Loesing
Date: Mon Nov 12 19:50:46 2018 +0100
Remove advbw column from bandwidth.csv.
Instead use advbw data from ipv6servers module.
As a result, we can stop aggregating advertised bandwidths in the
legacy module.
Required schema changes to live tordir databases:
DROP VIEW stats_bandwidth;
CREATE VIEW stats_bandwidth [...]
CREATE OR REPLACE FUNCTION refresh_all() [...]
DROP FUNCTION refresh_bandwidth_flags();
DROP FUNCTION refresh_relay_statuses_per_day();
DROP TABLE relay_statuses_per_day;
DROP TABLE bandwidth_flags;
DROP TABLE consensus;
DROP FUNCTION delete_old_descriptor();
DROP TABLE descriptor;
Part of #28116.
---
src/main/R/rserver/graphs.R| 58 +++---
.../metrics/stats/ipv6servers/Database.java| 22 ++
.../torproject/metrics/stats/ipv6servers/Main.java | 2 +
.../metrics/stats/servers/Configuration.java | 1 -
.../servers/RelayDescriptorDatabaseImporter.java | 232 +
src/main/sql/ipv6servers/init-ipv6servers.sql | 11 +
src/main/sql/legacy/tordir.sql | 135 +---
7 files changed, 73 insertions(+), 388 deletions(-)
diff --git a/src/main/R/rserver/graphs.R b/src/main/R/rserver/graphs.R
index 9dc8c2d..df108e2 100644
--- a/src/main/R/rserver/graphs.R
+++ b/src/main/R/rserver/graphs.R
@@ -446,16 +446,19 @@ write_platforms <- function(start_p = NULL, end_p = NULL,
path_p) {
}
prepare_bandwidth <- function(start_p, end_p) {
- read.csv(paste(stats_dir, "bandwidth.csv", sep = ""),
+ advbw <- read.csv(paste(stats_dir, "advbw.csv", sep = ""),
+colClasses = c("date" = "Date")) %>%
+transmute(date, variable = "advbw", value = advbw * 8 / 1e9)
+ bwhist <- read.csv(paste(stats_dir, "bandwidth.csv", sep = ""),
colClasses = c("date" = "Date")) %>%
+transmute(date, variable = "bwhist", value = (bwread + bwwrite) * 8 / 2e9)
+ rbind(advbw, bwhist) %>%
filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
-filter(isexit != "") %>%
-filter(isguard != "") %>%
-group_by(date) %>%
-summarize(advbw = sum(advbw) * 8 / 1e9,
- bwhist = sum(bwread + bwwrite) * 8 / 2e9) %>%
-select(date, advbw, bwhist)
+filter(!is.na(value)) %>%
+group_by(date, variable) %>%
+summarize(value = sum(value)) %>%
+spread(variable, value)
}
plot_bandwidth <- function(start_p, end_p, path_p) {
@@ -810,33 +813,24 @@ write_connbidirect <- function(start_p = NULL, end_p =
NULL, path_p) {
}
prepare_bandwidth_flags <- function(start_p, end_p) {
- b <- read.csv(paste(stats_dir, "bandwidth.csv", sep = ""),
-colClasses = c("date" = "Date"))
- b <- b %>%
+ advbw <- read.csv(paste(stats_dir, "advbw.csv", sep = ""),
+colClasses = c("date" = "Date")) %>%
+transmute(date, isguard, isexit, variable = "advbw",
+ value = advbw * 8 / 1e9)
+ bwhist <- read.csv(paste(stats_dir, "bandwidth.csv", sep = ""),
+colClasses = c("date" = "Date")) %>%
+transmute(date, isguard, isexit, variable = "bwhist",
+ value = (bwread + bwwrite) * 8 / 2e9)
+ rbind(advbw, bwhist) %>%
filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
-filter(isexit != "") %>%
-filter(isguard != "")
- b <- data.frame(date = b$date,
- isexit = b$isexit == "t", isguard = b$isguard == "t",
- advbw = b$advbw * 8 / 1e9,
- bwhist = (b$bwread + b$bwwrite) * 8 / 2e9)
- b <- rbind(
-data.frame(b[b$isguard == TRUE, ], flag = "guard"),
-data.frame(b[b$isexit == TRUE, ], flag = "exit"))
- b <- data.frame(date = b$date, advbw = b$advbw, bwhist = b$bwhist,
- flag = b$flag)
- b <- aggregate(list(advbw = b$advbw, bwhist = b$bwhist),
- by = list(date = b$date, flag = b$flag), FUN = sum,
- na.rm = TRUE, na.action = NULL)
- b <- gather(b, type, value, -c(date, flag))
- bandwidth <- b[b$value > 0, ]
- bandwidth <- data.frame(date = bandwidth$date,
-variable = as.factor(paste(bandwidth$flag, "_", bandwidth$type,
-sep = "")), value = bandwidth$value)
- bandwidth$variable <- factor(bandwidth$variable,
-levels = levels(bandwidth$variable)[c(3, 4, 1, 2)])
- bandwidth
+
[tor-commits] [metrics-web/release] Update to latest metrics-lib.
commit af54ce2d150c8508ca88ee15be6b79f4498d98b4 Author: Karsten Loesing Date: Mon May 13 17:26:58 2019 +0200 Update to latest metrics-lib. --- src/submods/metrics-lib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/submods/metrics-lib b/src/submods/metrics-lib index 23927c2..3693e10 16 --- a/src/submods/metrics-lib +++ b/src/submods/metrics-lib @@ -1 +1 @@ -Subproject commit 23927c2777f273c42ad3e75fc0a2940ed8eb4bf6 +Subproject commit 3693e107a3aff7473200ece3ba3889dc9462c7b3 ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Make very thin ribbons more visible.
commit 24aa37f07397bd1ba5022842eb5b2ce10d6ce6dd
Author: Karsten Loesing
Date: Wed Mar 6 11:13:48 2019 +0100
Make very thin ribbons more visible.
Fixes #29655.
---
src/main/R/rserver/rserve-init.R | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/main/R/rserver/rserve-init.R b/src/main/R/rserver/rserve-init.R
index 57e14f5..c412e4c 100644
--- a/src/main/R/rserver/rserve-init.R
+++ b/src/main/R/rserver/rserve-init.R
@@ -1029,7 +1029,7 @@ plot_userstats_bridge_combined <- function(start_p,
end_p, country_p, path_p) {
title <- paste("Bridge users by transport from ",
countryname(country_p), sep = "")
ggplot(u, aes(x = as.Date(date), ymin = low, ymax = high,
- fill = transport)) +
+ fill = transport, colour = transport)) +
geom_ribbon(alpha = 0.5, size = 0.5) +
scale_x_date(name = "", breaks = custom_breaks,
labels = custom_labels, minor_breaks = custom_minor_breaks) +
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Switch to readr's read_csv() everywhere.
commit a94a3844644041f7c1f6e0a4451e19ce12cae9e8
Author: Karsten Loesing
Date: Thu Jan 10 22:32:28 2019 +0100
Switch to readr's read_csv() everywhere.
---
src/main/R/rserver/graphs.R | 230 +---
1 file changed, 175 insertions(+), 55 deletions(-)
diff --git a/src/main/R/rserver/graphs.R b/src/main/R/rserver/graphs.R
index 82a51e7..205afbe 100644
--- a/src/main/R/rserver/graphs.R
+++ b/src/main/R/rserver/graphs.R
@@ -359,8 +359,11 @@ write_data <- function(FUN, ..., path_p) {
options(readr.show_progress = FALSE)
prepare_networksize <- function(start_p = NULL, end_p = NULL) {
- read.csv(paste(stats_dir, "networksize.csv", sep = ""),
-colClasses = c("date" = "Date")) %>%
+ read_csv(file = paste(stats_dir, "networksize.csv", sep = ""),
+ col_types = cols(
+date = col_date(format = ""),
+relays = col_double(),
+bridges = col_double())) %>%
filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE)
}
@@ -416,8 +419,11 @@ plot_versions <- function(start_p, end_p, path_p) {
}
prepare_platforms <- function(start_p = NULL, end_p = NULL) {
- read.csv(paste(stats_dir, "platforms.csv", sep = ""),
-colClasses = c("date" = "Date")) %>%
+ read_csv(file = paste(stats_dir, "platforms.csv", sep = ""),
+ col_types = cols(
+date = col_date(format = ""),
+platform = col_factor(levels = NULL),
+relays = col_double())) %>%
filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
mutate(platform = tolower(platform)) %>%
@@ -443,12 +449,19 @@ plot_platforms <- function(start_p, end_p, path_p) {
}
prepare_dirbytes <- function(start_p = NULL, end_p = NULL) {
- read.csv(paste(stats_dir, "bandwidth.csv", sep = ""),
-colClasses = c("date" = "Date")) %>%
+ read_csv(file = paste(stats_dir, "bandwidth.csv", sep = ""),
+ col_types = cols(
+date = col_date(format = ""),
+isexit = col_logical(),
+isguard = col_logical(),
+bwread = col_skip(),
+bwwrite = col_skip(),
+dirread = col_double(),
+dirwrite = col_double())) %>%
filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
-filter(isexit == "") %>%
-filter(isguard == "") %>%
+filter(is.na(isexit)) %>%
+filter(is.na(isguard)) %>%
mutate(dirread = dirread * 8 / 1e9,
dirwrite = dirwrite * 8 / 1e9) %>%
select(date, dirread, dirwrite)
@@ -473,8 +486,11 @@ plot_dirbytes <- function(start_p, end_p, path_p) {
}
prepare_relayflags <- function(start_p = NULL, end_p = NULL, flag_p = NULL) {
- read.csv(paste(stats_dir, "relayflags.csv", sep = ""),
-colClasses = c("date" = "Date")) %>%
+ read_csv(file = paste(stats_dir, "relayflags.csv", sep = ""),
+ col_types = cols(
+date = col_date(format = ""),
+flag = col_factor(levels = NULL),
+relays = col_double())) %>%
filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
filter(if (!is.null(flag_p)) flag %in% flag_p else TRUE)
@@ -483,7 +499,7 @@ prepare_relayflags <- function(start_p = NULL, end_p =
NULL, flag_p = NULL) {
plot_relayflags <- function(start_p, end_p, flag_p, path_p) {
prepare_relayflags(start_p, end_p, flag_p) %>%
complete(date = full_seq(date, period = 1), flag = unique(flag)) %>%
-ggplot(aes(x = date, y = relays, colour = as.factor(flag))) +
+ggplot(aes(x = date, y = relays, colour = flag)) +
geom_line() +
scale_x_date(name = "", breaks = custom_breaks,
labels = custom_labels, minor_breaks = custom_minor_breaks) +
@@ -498,8 +514,18 @@ plot_relayflags <- function(start_p, end_p, flag_p,
path_p) {
prepare_torperf <- function(start_p = NULL, end_p = NULL, server_p = NULL,
filesize_p = NULL) {
- read.csv(paste(stats_dir, "torperf-1.1.csv", sep = ""),
-colClasses = c("date" = "Date", "source" = "character")) %>%
+ read_csv(file = paste(stats_dir, "torperf-1.1.csv", sep = ""),
+ col_types = cols(
+date = col_date(format = ""),
+filesize = col_double(),
+source = col_character(),
+server = col_character(),
+q1
[tor-commits] [metrics-web/release] Run modules from Java only.
commit 829863f48fb4e624f849df4c67bc970331014b0d
Author: Karsten Loesing
Date: Thu Jan 24 10:08:02 2019 +0100
Run modules from Java only.
Implements #29166.
---
build.xml | 136 +
.../web/images => R/rserver}/no-data-available.pdf | Bin
.../web/images => R/rserver}/no-data-available.png | Bin
.../web/images => R/rserver}/no-data-available.xcf | Bin
src/main/R/rserver/rserve-init.R | 6 +-
.../torproject/metrics/stats/advbwdist/Main.java | 15 ++-
.../metrics/stats/bwhist/Configuration.java| 18 ---
.../org/torproject/metrics/stats/bwhist/Main.java | 31 +++--
.../torproject/metrics/stats/clients/Database.java | 10 +-
.../torproject/metrics/stats/clients/Detector.java | 7 +-
.../org/torproject/metrics/stats/clients/Main.java | 37 +++---
.../metrics/stats/collectdescs/Main.java | 5 +-
.../metrics/stats/connbidirect/Main.java | 15 ++-
.../org/torproject/metrics/stats/hidserv/Main.java | 14 ++-
.../org/torproject/metrics/stats/main/Main.java| 122 ++
.../torproject/metrics/stats/onionperf/Main.java | 26 ++--
.../metrics/stats/servers/Configuration.java | 18 ---
.../torproject/metrics/stats/servers/Database.java | 10 +-
.../org/torproject/metrics/stats/servers/Main.java | 45 +++
.../metrics/stats/totalcw/Configuration.java | 18 ---
.../torproject/metrics/stats/totalcw/Database.java | 10 +-
.../org/torproject/metrics/stats/totalcw/Main.java | 29 +++--
.../torproject/metrics/stats/webstats/Main.java| 23 ++--
23 files changed, 296 insertions(+), 299 deletions(-)
diff --git a/build.xml b/build.xml
index 42965bf..254a71f 100644
--- a/build.xml
+++ b/build.xml
@@ -18,7 +18,8 @@
+value="org.torproject.metrics.stats.main.Main" />
+
@@ -26,11 +27,6 @@
-
-
-
@@ -301,85 +297,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/src/main/resources/web/images/no-data-available.pdf
b/src/main/R/rserver/no-data-available.pdf
similarity index 100%
rename from src/main/resources/web/images/no-data-available.pdf
rename to src/main/R/rserver/no-data-available.pdf
diff --git a/src/main/resources/web/images/no-data-available.png
b/src/main/R/rserver/no-data-available.png
similarity index 100%
rename from src/main/resources/web/images/no-data-available.png
rename to src/main/R/rserver/no-data-available.png
diff --git a/src/main/resources/web/images/no-data-available.xcf
b/src/main/R/rserver/no-data-available.xcf
similarity index 100%
rename from src/main/resources/web/images/no-data-available.xcf
rename to src/main/R/rserver/no-data-available.xcf
diff --git a/src/main/R/rserver/rserve-init.R b/src/main/R/rserver/rserve-init.R
index c412e4c..5a47550 100644
--- a/src/main/R/rserver/rserve-init.R
+++ b/src/main/R/rserver/rserve-init.R
@@ -333,7 +333,7 @@ copyright_notice <- "The Tor Project -
https://metrics.torproject.org/;
stats_dir <- "/srv/metrics.torproject.org/metrics/shared/stats/"
-rdata_dir <- "/srv/metrics.torproject.org/metrics/shared/RData/"
+no_data_available_dir <-
"/srv/metrics.torproject.org/metrics/src/main/R/rserver/"
# Helper function that copies the appropriate no data object to filename.
copy_no_data <- function(filename) {
@@ -342,8 +342,8 @@ copy_no_data <- function(filename) {
if (".csv" == extension) {
write("# No data available for the given parameters.", file=filename)
} else {
-file.copy(paste(rdata_dir, "no-data-available", extension, sep = ""),
- filename)
+file.copy(paste(no_data_available_dir, "no-data-available", extension,
+ sep = ""), filename)
}
}
diff --git a/src/main/java/org/torproject/metrics/stats/advbwdist/Main.java
b/src/main/java/org/torproject/metrics/stats/advbwdist/Main.java
index 6c4f4ac..3afcb80 100644
--- a/src/main/java/org/torproject/metrics/stats/advbwdist/Main.java
+++ b/src/main/java/org/torproject/metrics/stats/advbwdist/Main.java
@@ -32,6 +32,9 @@ import java.util.TreeMap;
public class Main {
+ private static final File baseDir = new File(
+ org.torproject.metrics.stats.main.Main.modulesDir, "advbwdist");
+
/** Executes this data-processing module. */
public static void main(String[] args) throws IOException {
@@ -41,7 +44,8 @@ public
[tor-commits] [metrics-web/release] Re-add missing COMMIT commands to bwhist module.
commit 9dd35e29084ed9380cb374c80a4f9bfb0d9a91e2
Author: Karsten Loesing
Date: Thu Dec 20 11:28:46 2018 +0100
Re-add missing COMMIT commands to bwhist module.
Last month, in commit f8fa108 where we modernized the legacy module
and renamed it to bwhist, we split up the closeConnection() into one
method commit() to commit changes and another method closeConnection()
to close the connection. However, we somehow forgot to invoke the
commit() method.
This had two effects:
1. Newly added data was not made persistent in the database. This
lead to a moving window of roughly one week for new data and an
increasing gap between the last committed data and this 1-week
window.
2. The result of aggregating newly added data was not made
persistent. So, even after fixing the first issue above, we
accumulated newly added data, rather than only keeping the most
recent two weeks. This made the database slower over time.
This change adds two commit() calls at the right places.
---
.../metrics/stats/bwhist/RelayDescriptorDatabaseImporter.java | 2 ++
1 file changed, 2 insertions(+)
diff --git
a/src/main/java/org/torproject/metrics/stats/bwhist/RelayDescriptorDatabaseImporter.java
b/src/main/java/org/torproject/metrics/stats/bwhist/RelayDescriptorDatabaseImporter.java
index a6cf0cc..9f9ecff 100644
---
a/src/main/java/org/torproject/metrics/stats/bwhist/RelayDescriptorDatabaseImporter.java
+++
b/src/main/java/org/torproject/metrics/stats/bwhist/RelayDescriptorDatabaseImporter.java
@@ -532,6 +532,7 @@ public final class RelayDescriptorDatabaseImporter {
this.addExtraInfoDescriptor((ExtraInfoDescriptor) descriptor);
}
}
+this.commit();
reader.saveHistoryFile(this.historyFile);
}
@@ -615,6 +616,7 @@ public final class RelayDescriptorDatabaseImporter {
void aggregate() throws SQLException {
Statement st = this.conn.createStatement();
st.executeQuery("SELECT refresh_all()");
+this.commit();
}
/** Query the servers_platforms view. */
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Add three new @type annotations.
commit e7da8f637cbdd2d851d06657f7959befbb68dcf1 Author: Karsten Loesing Date: Thu Jan 24 11:27:29 2019 +0100 Add three new @type annotations. Resolves #28615. --- src/main/resources/web/jsps/collector.jsp | 58 +++ 1 file changed, 58 insertions(+) diff --git a/src/main/resources/web/jsps/collector.jsp b/src/main/resources/web/jsps/collector.jsp index 3e98785..1adc9b3 100644 --- a/src/main/resources/web/jsps/collector.jsp +++ b/src/main/resources/web/jsps/collector.jsp @@ -92,6 +92,11 @@ archive + Detached Signatures + @type detached-signature-3 1.0 + + + Microdescriptor Consensuses @type network-status-microdesc-consensus-3 1.0 recent @@ -104,6 +109,11 @@ archive + Network Status Entries + @type network-status-entry-3 1.0 + + + Version 2 Network Statuses @type network-status-2 1.0 archive @@ -177,6 +187,14 @@ recent archive + + Bandwidth Files + + + Bandwidth Files + @type bandwidth-file 1.2 + + @@ -298,6 +316,17 @@ These key certificates change once every few months, so they are only available in a single descriptor archive tarball. +Detached Signatures +@type detached-signature-3 1.0 +# + + + +Detached signature as per section 3.10 of the dir-spec, and downloadable for +DistSeconds every consensus freshness period (usually five minutes each hour) +via the /tor/status-vote/next/consensus-signatures resource. + + Microdescriptor Consensuses @type network-status-microdesc-consensus-3 1.0 recent @@ -335,6 +364,17 @@ contain all descriptors collected in an hour concatenated into a single file. +Network Status Entries +@type network-status-entry-3 1.0 +# + + + +Individual router status entry from an unflavored v3 network status document. +These are available from Tor's control port GETINFO ns/* commands +and NS events. + + Version 2 Network Statuses @type network-status-2 1.0 archive @@ -732,6 +772,24 @@ detail on a separate page. + +Bandwidth Files +# + + +Bandwidth Files +@type bandwidth-file 1.2 +# + + + +Bandwidth authority metrics as defined in the https://gitweb.torproject.org/torspec.git/tree/bandwidth-file-spec.txt;>bandwidth-file-spec. +These are available from a DirPort's +/tor/status-vote/next/bandwidth url and CollecTor (both are +https://trac.torproject.org/projects/tor/ticket/21377;>pending). + + + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Make bandwidth.csv independent of import order.
commit 1fd062c11e45ca5c4cb166f6db8a27eabaf60e48 Author: Karsten Loesing Date: Fri Mar 8 15:08:08 2019 +0100 Make bandwidth.csv independent of import order. With this patch we're not overwriting bandwidth history parts with whichever history comes last, but we're computing the maximum value for each 15-minute interval of all imported bandwidth histories. This makes bandwidth.csv independent of descriptor import order. --- src/main/sql/bwhist/tordir.sql | 41 ++--- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/src/main/sql/bwhist/tordir.sql b/src/main/sql/bwhist/tordir.sql index dfe7b5d..047f18e 100644 --- a/src/main/sql/bwhist/tordir.sql +++ b/src/main/sql/bwhist/tordir.sql @@ -122,6 +122,24 @@ CREATE TABLE updates ( date DATE ); +-- Return an array as the result of merging two arrays: if an array index is +-- only contained in one array, that array element is included in the result; +-- if an array index is contained in both arrays, the greater of the two +-- elements is included. +CREATE OR REPLACE FUNCTION array_merge(first BIGINT[], second BIGINT[]) +RETURNS BIGINT[] AS $$ +DECLARE + merged BIGINT[]; +BEGIN + FOR i IN LEAST(array_lower(first, 1), array_lower(second, 1)).. + GREATEST(array_upper(first, 1), array_upper(second, 1)) LOOP +merged[i] := GREATEST(first[i], second[i]); + END LOOP; +RETURN merged; +END; +$$ LANGUAGE plpgsql +STABLE RETURNS NULL ON NULL INPUT; + CREATE OR REPLACE FUNCTION array_sum (BIGINT[]) RETURNS BIGINT AS $$ SELECT SUM($1[i])::bigint FROM generate_series(array_lower($1, 1), array_upper($1, 1)) index(i); @@ -143,25 +161,10 @@ CREATE OR REPLACE FUNCTION insert_bwhist( ELSE BEGIN UPDATE bwhist -SET read[array_lower(insert_read, 1): - array_upper(insert_read, 1)] = insert_read, -written[array_lower(insert_written, 1): - array_upper(insert_written, 1)] = insert_written, -dirread[array_lower(insert_dirread, 1): - array_upper(insert_dirread, 1)] = insert_dirread, -dirwritten[array_lower(insert_dirwritten, 1): - array_upper(insert_dirwritten, 1)] = insert_dirwritten -WHERE fingerprint = insert_fingerprint AND date = insert_date; --- Updating twice is an ugly workaround for PostgreSQL bug 5840 -UPDATE bwhist -SET read[array_lower(insert_read, 1): - array_upper(insert_read, 1)] = insert_read, -written[array_lower(insert_written, 1): - array_upper(insert_written, 1)] = insert_written, -dirread[array_lower(insert_dirread, 1): - array_upper(insert_dirread, 1)] = insert_dirread, -dirwritten[array_lower(insert_dirwritten, 1): - array_upper(insert_dirwritten, 1)] = insert_dirwritten +SET read = array_merge(read, insert_read), +written = array_merge(written, insert_written), +dirread = array_merge(dirread, insert_dirread), +dirwritten = array_merge(dirwritten, insert_dirwritten) WHERE fingerprint = insert_fingerprint AND date = insert_date; END; END IF; ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Simplify plot_webstats_tb_locale function.
commit 2b34cd2023a3e59057f4274afb0d7b8163282a18
Author: Karsten Loesing
Date: Thu Jan 10 10:41:48 2019 +0100
Simplify plot_webstats_tb_locale function.
---
src/main/R/rserver/graphs.R | 61 -
1 file changed, 27 insertions(+), 34 deletions(-)
diff --git a/src/main/R/rserver/graphs.R b/src/main/R/rserver/graphs.R
index ba8862c..27f399d 100644
--- a/src/main/R/rserver/graphs.R
+++ b/src/main/R/rserver/graphs.R
@@ -1265,8 +1265,8 @@ write_webstats_tb_platform <- function(start_p = NULL,
end_p = NULL, path_p) {
write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}
-plot_webstats_tb_locale <- function(start_p, end_p, path_p) {
- d <- read_csv(file = paste(stats_dir, "webstats.csv", sep = ""),
+prepare_webstats_tb_locale <- function(start_p, end_p) {
+ read_csv(file = paste(stats_dir, "webstats.csv", sep = ""),
col_types = cols(
log_date = col_date(format = ""),
request_type = col_factor(),
@@ -1274,20 +1274,35 @@ plot_webstats_tb_locale <- function(start_p, end_p,
path_p) {
channel = col_skip(),
locale = col_factor(),
incremental = col_skip(),
-count = col_double()))
- d <- d[d$log_date >= start_p & d$log_date <= end_p &
- d$request_type %in% c("tbid", "tbup"), ]
- levels(d$request_type) <- list(
- "Initial downloads" = "tbid",
- "Update pings" = "tbup")
+count = col_double())) %>%
+filter(if (!is.null(start_p)) log_date >= as.Date(start_p) else TRUE) %>%
+filter(if (!is.null(end_p)) log_date <= as.Date(end_p) else TRUE) %>%
+filter(request_type %in% c("tbid", "tbup")) %>%
+rename(date = log_date) %>%
+group_by(date, locale, request_type) %>%
+summarize(count = sum(count)) %>%
+mutate(request_type = factor(request_type, levels = c("tbid", "tbup"))) %>%
+spread(request_type, count, fill = 0) %>%
+rename(initial_downloads = tbid, update_pings = tbup)
+}
+
+plot_webstats_tb_locale <- function(start_p, end_p, path_p) {
+ d <- prepare_webstats_tb_locale(start_p, end_p) %>%
+gather(request_type, count, -c(date, locale)) %>%
+mutate(request_type = factor(request_type,
+ levels = c("initial_downloads", "update_pings"),
+ labels = c("Initial downloads", "Update pings")))
e <- d
e <- aggregate(list(count = e$count), by = list(locale = e$locale), FUN =
sum)
e <- e[order(e$count, decreasing = TRUE), ]
e <- e[1:5, ]
- d <- aggregate(list(count = d$count), by = list(log_date = d$log_date,
+ d <- aggregate(list(count = d$count), by = list(date = d$date,
request_type = d$request_type,
locale = ifelse(d$locale %in% e$locale, d$locale, "(other)")), FUN = sum)
- ggplot(d, aes(x = log_date, y = count, colour = locale)) +
+ d %>%
+complete(date = full_seq(date, period = 1),
+ nesting(locale, request_type)) %>%
+ggplot(aes(x = date, y = count, colour = locale)) +
geom_point() +
geom_line() +
scale_x_date(name = "", breaks = custom_breaks,
@@ -1295,7 +1310,7 @@ plot_webstats_tb_locale <- function(start_p, end_p,
path_p) {
scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) +
scale_colour_hue(name = "Locale",
breaks = c(e$locale, "(other)"),
-labels = c(e$locale, "Other")) +
+labels = c(as.character(e$locale), "Other")) +
facet_grid(request_type ~ ., scales = "free_y") +
theme(strip.text.y = element_text(angle = 0, hjust = 0, size = rel(1.5)),
strip.background = element_rect(fill = NA),
@@ -1305,30 +1320,8 @@ plot_webstats_tb_locale <- function(start_p, end_p,
path_p) {
ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}
-# Ideally, this function would share code with plot_webstats_tb_locale
-# by using a common prepare_webstats_tb_locale function. This just
-# turned out to be a bit harder than for other functions, because
-# plot_webstats_tb_locale needs the preliminary data frame e for its
-# breaks and labels. Left as future work.
write_webstats_tb_locale <- function(start_p = NULL, end_p = NULL, path_p) {
- read_csv(file = paste(stats_dir, "webstats.csv", sep = ""),
- col_types = cols(
-log_date = col_date(format = ""),
-request_type = col_factor(),
-platform = col_skip(),
-channel = col_skip(),
-locale = col_factor(),
-incremental = col_skip(),
-count = col_double())) %>%
-filter(if (!is.null(start_p)) log_date >= as.Date(start_p) else TRUE) %>%
-filter(if (!is.null(end_p)) l
[tor-commits] [metrics-web/release] Stop calling censorship detector BETA.
commit 8c24f8e174c277e1c25222bbd598bb2278289e00
Author: Karsten Loesing
Date: Mon Jan 7 12:20:35 2019 +0100
Stop calling censorship detector BETA.
---
src/main/resources/web/jsps/graph.jsp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/src/main/resources/web/jsps/graph.jsp
b/src/main/resources/web/jsps/graph.jsp
index e710d2c..baa7edb 100644
--- a/src/main/resources/web/jsps/graph.jsp
+++ b/src/main/resources/web/jsps/graph.jsp
@@ -93,8 +93,7 @@
-Show possible censorship events if available (http://research.torproject.org/techreports/detector-2011-09-09.pdf;>BETA):
+Show possible http://research.torproject.org/techreports/detector-2011-09-09.pdf;>censorship
events if available:
${row[2]}
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Document changes to OnionPerf graphs.
commit ad1221cb980aa5bf3bf075338d2588d803e652c2 Author: Karsten Loesing Date: Thu Dec 20 10:03:31 2018 +0100 Document changes to OnionPerf graphs. Still related to #28603. --- src/main/resources/web/jsps/stats.jsp | 14 +- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/main/resources/web/jsps/stats.jsp b/src/main/resources/web/jsps/stats.jsp index e5f9c6a..2ae6726 100644 --- a/src/main/resources/web/jsps/stats.jsp +++ b/src/main/resources/web/jsps/stats.jsp @@ -48,7 +48,7 @@ https://metrics.torproject.org/identifier.csv August 15, 2018: Made the first batch of changes to per-graph CSV files. September 15, 2018: Removed all pre-aggregated CSV files. October 28, 2018: Added and/or removed columns to Tor Browser downloads and updates by platform and Tor Browser downloads and updates by locale graphs. -December 20, 2018 (scheduled): Remove source parameters and output rows with aggregates over all sources from Time to download files over Tor, Timeouts and failures of downloading files over Tor, Circuit build times, Circuit round-trip latencies graphs. +December 20, 2018: Removed source parameters and output rows with aggregates over all sources from Time to download files over Tor, Timeouts and failures of downloading files over Tor, Circuit build times, Circuit round-trip latencies graphs. December 20, 2018 (scheduled): Remove two graphs Total relay bandwidth and Consumed bandwidth by Exit/Guard flag combination, and update the data format of the Advertised and consumed bandwidth by relay flag graph to cover all data previously contained in the first two graphs. @@ -536,7 +536,6 @@ Performance # start: First UTC date (-MM-DD) to include in the file. end: Last UTC date (-MM-DD) to include in the file. -source: Name of the OnionPerf or Torperf service performing measurements, or "all" for measurements performed by any service. This parameter is going to be removed after December 20, 2018. server: Either "public" for requests to a server on the public internet, or "onion" for requests to a version 2 onion server. filesize: Size of the downloaded file in bytes, with pre-defined possible values: "50kb", "1mb", or "5mb". @@ -546,7 +545,7 @@ Performance # date: UTC date (-MM-DD) when download performance was measured. filesize: Size of the downloaded file in bytes. -source: Name of the OnionPerf or Torperf service performing measurements. If this column contains the empty string, all measurements are included, regardless of which service performed them. Output rows with aggregates over all sources are going to be removed after December 20, 2018. +source: Name of the OnionPerf or Torperf service performing measurements. server: Either "public" if the request was made to a server on the public internet, or "onion" if the request was made to a version 2 onion server. q1: First quartile of time in milliseconds until receiving the last byte. md: Median of time in milliseconds until receiving the last byte. @@ -563,7 +562,6 @@ Performance # start: First UTC date (-MM-DD) to include in the file. end: Last UTC date (-MM-DD) to include in the file. -source: Name of the OnionPerf or Torperf service performing measurements, or "all" for measurements performed by any service. This parameter is going to be removed after December 20, 2018. server: Either "public" for requests to a server on the public internet, or "onion" for requests to a version 2 onion server. filesize: Size of the downloaded file in bytes, with pre-defined possible values: "50kb", "1mb", or "5mb". @@ -573,7 +571,7 @@ Performance # date: UTC date (-MM-DD) when download performance was measured. filesize: Size of the downloaded file in bytes. -source: Name of the OnionPerf or Torperf service performing measurements. If this column contains the empty string, all measurements are included, regardless of which service performed them. Output rows with aggregates over all sources are going to be removed after December 20, 2018. +source: Name of the OnionPerf or Torperf service performing measurements. server: Either "public" if the request was made to a server on the public internet, or "onion" if the request was made to a version 2 onion server. timeouts: Fraction of requests that timed out when attempting to download the static file over Tor. failures: Fraction of requests that failed when attempting to download the static file over Tor. @@ -589,14 +587,13 @@ Performance # start: First UTC date (-MM-DD) to include in the file. end: Last UTC date (-MM-DD) to include in the file. -source: Name of the OnionPerf or Torperf service performing measurements, or "all" for measurements performed by any service. This parameter is goin
[tor-commits] [metrics-web/release] Stop hard-coding versions.
commit c8a3414347c8df7aed3e63be4c704f1dd43aded4
Author: Karsten Loesing
Date: Mon Jan 7 09:59:18 2019 +0100
Stop hard-coding versions.
---
src/main/R/rserver/graphs.R | 12 ++--
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/src/main/R/rserver/graphs.R b/src/main/R/rserver/graphs.R
index 03b5b93..d3ea90a 100644
--- a/src/main/R/rserver/graphs.R
+++ b/src/main/R/rserver/graphs.R
@@ -381,18 +381,18 @@ write_networksize <- function(start_p = NULL, end_p =
NULL, path_p) {
}
prepare_versions <- function(start_p, end_p) {
- read.csv(paste(stats_dir, "versions.csv", sep = ""),
-colClasses = c("date" = "Date")) %>%
+ read_csv(paste(stats_dir, "versions.csv", sep = ""),
+ col_types = cols(
+date = col_date(format = ""),
+version = col_character(),
+relays = col_double())) %>%
filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE)
}
plot_versions <- function(start_p, end_p, path_p) {
s <- prepare_versions(start_p, end_p)
- known_versions <- c("Other", "0.1.0", "0.1.1", "0.1.2", "0.2.0",
-"0.2.1", "0.2.2", "0.2.3", "0.2.4", "0.2.5", "0.2.6", "0.2.7",
-"0.2.8", "0.2.9", "0.3.0", "0.3.1", "0.3.2", "0.3.3", "0.3.4",
-"0.3.5")
+ known_versions <- unique(s$version)
getPalette <- colorRampPalette(brewer.pal(12, "Paired"))
colours <- data.frame(breaks = known_versions,
values = rep(brewer.pal(min(12, length(known_versions)), "Paired"),
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Document another change to OnionPerf graphs.
commit 0ca51404b70258e1594df2d88f460fd6a6daf351 Author: Karsten Loesing Date: Thu Dec 20 10:33:00 2018 +0100 Document another change to OnionPerf graphs. Still related to #28603. --- src/main/resources/web/jsps/stats.jsp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/resources/web/jsps/stats.jsp b/src/main/resources/web/jsps/stats.jsp index 2ae6726..a78da8a 100644 --- a/src/main/resources/web/jsps/stats.jsp +++ b/src/main/resources/web/jsps/stats.jsp @@ -610,6 +610,7 @@ Performance # start: First UTC date (-MM-DD) to include in the file. end: Last UTC date (-MM-DD) to include in the file. +server: Either "public" for requests to a server on the public internet, or "onion" for requests to a version 2 onion server. Columns ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Properly skip previously imported webstats files.
commit 9bdb6d39fc7b0ac8e7327caeafabfac43a41689f
Author: Karsten Loesing
Date: Mon Jan 7 11:59:19 2019 +0100
Properly skip previously imported webstats files.
Turns out we never skipped previously imported webstats files due to
two bugs:
1. While building a list of previously imported webstats files we
reassembled their file names as ${server}_${site}_* rather than
${site}_${server}_* which was the file name format we chose in an
earlier version of the CollecTor module.
2. When checking whether a given webstats file already exists in the
database we compared the full file name to the reassembled file
name from the database with ${server} being truncated to 32
characters.
This commit fixes both bugs.
---
src/main/java/org/torproject/metrics/stats/webstats/Main.java | 10 --
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/src/main/java/org/torproject/metrics/stats/webstats/Main.java
b/src/main/java/org/torproject/metrics/stats/webstats/Main.java
index a154e64..fb0a903 100644
--- a/src/main/java/org/torproject/metrics/stats/webstats/Main.java
+++ b/src/main/java/org/torproject/metrics/stats/webstats/Main.java
@@ -100,7 +100,7 @@ public class Main {
try (ResultSet rs = st.executeQuery(queryString)) {
while (rs.next()) {
importedLogFileUrls.add(String.format("%s_%s_access.log_%s.xz",
-rs.getString(1), rs.getString(2),
+rs.getString(2), rs.getString(1),
rs.getDate(3).toLocalDate().format(dateFormat)));
}
}
@@ -111,13 +111,19 @@ public class Main {
static void importLogFiles(Connection connection, SortedSet
skipFiles,
File... inDirectories) {
+DateTimeFormatter dateFormat = DateTimeFormatter.ofPattern("MMdd");
for (Descriptor descriptor : DescriptorSourceFactory
.createDescriptorReader().readDescriptors(inDirectories)) {
if (!(descriptor instanceof WebServerAccessLog)) {
continue;
}
WebServerAccessLog logFile = (WebServerAccessLog) descriptor;
- if (skipFiles.contains(logFile.getDescriptorFile().getName())) {
+ String logFileNameWithTruncatedParts = String.format(
+ "%s_%s_access.log_%s.xz",
+ truncateString(logFile.getVirtualHost(), 128),
+ truncateString(logFile.getPhysicalHost(), 32),
+ logFile.getLogDate().format(dateFormat));
+ if (skipFiles.contains(logFileNameWithTruncatedParts)) {
continue;
}
try {
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Rewrite advbwdist's aggregate.R in Java.
commit b605298c66c8c348fe589062dc1ddd3da293c8db
Author: Karsten Loesing
Date: Mon Dec 10 15:17:46 2018 +0100
Rewrite advbwdist's aggregate.R in Java.
This is yet another step torwards making the daily update Java-only.
Implements #28801.
---
build.xml | 17
src/main/R/advbwdist/aggregate.R | 25
.../torproject/metrics/stats/advbwdist/Main.java | 46 ++
3 files changed, 46 insertions(+), 42 deletions(-)
diff --git a/build.xml b/build.xml
index 93eda7b..42965bf 100644
--- a/build.xml
+++ b/build.xml
@@ -347,9 +347,6 @@
-
-
-
@@ -453,20 +450,6 @@
-
-
-
-
-
-
-
-
-
-
-
diff --git a/src/main/R/advbwdist/aggregate.R b/src/main/R/advbwdist/aggregate.R
deleted file mode 100644
index 1c67dff..000
--- a/src/main/R/advbwdist/aggregate.R
+++ /dev/null
@@ -1,25 +0,0 @@
-require(reshape)
-t <- read.csv("stats/advbwdist-validafter.csv",
- colClasses = c("character", "logical", "integer", "integer", "integer"),
- stringsAsFactors = FALSE)
-
-currSysDate <- paste(Sys.Date() - 1, "23:59:59")
-t <- t[t$valid_after < currSysDate, ]
-t$date <- as.factor(substr(t$valid_after, 1, 10))
-t$isexit <- !is.na(t$isexit)
-t$relay <- ifelse(is.na(t$relay), -1, t$relay)
-t$percentile <- ifelse(is.na(t$percentile), -1, t$percentile)
-
-t <- aggregate(list(advbw = t$advbw), by = list(date = t$date,
-isexit = t$isexit, relay = t$relay, percentile = t$percentile),
-FUN = median)
-
-t$isexit <- ifelse(t$isexit, "t", "")
-t$relay <- ifelse(t$relay < 0, NA, t$relay)
-t$percentile <- ifelse(t$percentile < 0, NA, t$percentile)
-t$advbw <- floor(t$advbw)
-
-t <- t[order(t$date, t$isexit, t$relay, t$percentile), ]
-
-write.csv(t, "stats/advbwdist.csv", quote = FALSE, row.names = FALSE, na = "")
-
diff --git a/src/main/java/org/torproject/metrics/stats/advbwdist/Main.java
b/src/main/java/org/torproject/metrics/stats/advbwdist/Main.java
index 7216581..6c4f4ac 100644
--- a/src/main/java/org/torproject/metrics/stats/advbwdist/Main.java
+++ b/src/main/java/org/torproject/metrics/stats/advbwdist/Main.java
@@ -10,15 +10,19 @@ import org.torproject.descriptor.NetworkStatusEntry;
import org.torproject.descriptor.RelayNetworkStatusConsensus;
import org.torproject.descriptor.ServerDescriptor;
+import org.apache.commons.math3.stat.descriptive.rank.Median;
import org.apache.commons.math3.stat.descriptive.rank.Percentile;
+import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
+import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -138,6 +142,39 @@ public class Main {
}
descriptorReader.saveHistoryFile(historyFile);
bw.close();
+
+/* Aggregate statistics. */
+SimpleDateFormat dateFormat = new SimpleDateFormat("-MM-dd");
+dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+String today = dateFormat.format(new Date());
+SortedMap> preAggregatedValues = new TreeMap<>();
+try (BufferedReader br = new BufferedReader(new FileReader(resultsFile))) {
+ br.readLine(); /* Skip header. */
+ String line;
+ while (null != (line = br.readLine())) {
+String[] parts = line.split(",");
+String date = parts[0].substring(0, 10);
+if (date.compareTo(today) >= 0) {
+ continue;
+}
+String isExit = parts[1].equals("TRUE") ? "t" : "";
+String keyWithoutTime = String.format("%s,%s,%s,%s",
+date, isExit, parts[2], parts[3]);
+long value = Long.parseLong(parts[4]);
+preAggregatedValues.putIfAbsent(keyWithoutTime, new ArrayList<>());
+preAggregatedValues.get(keyWithoutTime).add(value);
+ }
+}
+File aggregateResultsFile = new File("stats/advbwdist.csv");
+aggregateResultsFile.getParentFile().mkdirs();
+try (BufferedWriter bw2 = new BufferedWriter(
+new FileWriter(aggregateResultsFile))) {
+ bw2.write("date,isexit,relay,percentile,advbw\n");
+ for (Map.Entry> e : preAggregatedValues.entrySet()) {
+bw2.write(String.format("%s,%.0f%n", e.getKey(),
+computeMedian(e.getValue(;
+ }
+}
}
/** Compute percentiles (between 0 and 100) for the given list of values, and
@@ -168,5 +205,14 @@ public class Main {
}
return computedPercentiles;
}
+
+ /** Return the median for the given li
[tor-commits] [metrics-web/release] Update news.json to version 307 of doc/MetricsTimeline.
commit f6f0570819a8a2e05c3e22636b21e00629d50b8f
Author: Karsten Loesing
Date: Mon Jan 7 12:27:51 2019 +0100
Update news.json to version 307 of doc/MetricsTimeline.
---
src/main/resources/web/json/news.json | 202 +-
1 file changed, 173 insertions(+), 29 deletions(-)
diff --git a/src/main/resources/web/json/news.json
b/src/main/resources/web/json/news.json
index 9810770..26bf78a 100644
--- a/src/main/resources/web/json/news.json
+++ b/src/main/resources/web/json/news.json
@@ -2489,6 +2489,15 @@
"target" : "https://en.wikipedia.org/wiki/Hurricane_Maria#Puerto_Rico_3;
} ]
}, {
+ "start" : "2017-10-04",
+ "protocols" : [ "fte" ],
+ "short_description" : "Permanent hardware failure of default FTE bridge
128.105.214.161:8080.",
+ "description" : "Permanent hardware failure of default FTE bridge
128.105.214.161:8080.",
+ "links" : [ {
+"label" : "comment",
+"target" : "https://bugs.torproject.org/28521#comment:2;
+ } ]
+}, {
"start" : "2017-10-05",
"protocols" : [ "ipv4", "ipv6" ],
"short_description" : "geoip and geoip6 databases updated to \"October 4
2017 Maxmind GeoLite2 Country\"",
@@ -2534,6 +2543,9 @@
"links" : [ {
"label" : "wikipedia",
"target" :
"https://en.wikipedia.org/wiki/19th_National_Congress_of_the_Communist_Party_of_China;
+ }, {
+"label" : "Psiphon users",
+"target" :
"https://media.ccc.de/v/35c3-9964-cat_mouse_evading_the_censors_in_2018#t=2067;
} ]
}, {
"start" : "2017-10-25",
@@ -2754,6 +2766,9 @@
}, {
"label" : "tweet",
"target" : "https://twitter.com/nusenu_/status/948914485045145601;
+ }, {
+"label" : "Psiphon users",
+"target" :
"https://media.ccc.de/v/35c3-9964-cat_mouse_evading_the_censors_in_2018#t=1770;
} ]
}, {
"start" : "2018-01-01",
@@ -3045,6 +3060,16 @@
"target" :
"https://metrics.torproject.org/userstats-bridge-country.html?start=2018-03-01=2018-07-15=ru;
} ]
}, {
+ "start" : "2018-04-20",
+ "end" : "2018-04-27",
+ "protocols" : [ "onion" ],
+ "short_description" : "The number of v2 onion services increases from 70k to
120k.",
+ "description" : "The number of v2 onion services increases from 70k to
120k.",
+ "links" : [ {
+"label" : "ticket",
+"target" : "https://bugs.torproject.org/26081;
+ } ]
+}, {
"start" : "2018-04-28",
"places" : [ "ru" ],
"short_description" : "Russia unblocks about 3 million IP addresses
belonging to Amazon and OVH.",
@@ -3074,6 +3099,9 @@
}, {
"label" : "bridge graph",
"target" :
"https://metrics.torproject.org/userstats-bridge-country.html?start=2018-03-11=2018-06-01=ir;
+ }, {
+"label" : "Psiphon users",
+"target" :
"https://media.ccc.de/v/35c3-9964-cat_mouse_evading_the_censors_in_2018#t=1922;
} ]
}, {
"start" : "2018-05-08",
@@ -3169,6 +3197,16 @@
"target" : "https://www.accessnow.org/venezuela-blocks-tor/;
} ]
}, {
+ "start" : "2018-06-24",
+ "end" : "2018-07-06",
+ "protocols" : [ "snowflake" ],
+ "short_description" : "Metrics for Snowflake are missing, for unknown
reasons",
+ "description" : "Metrics for Snowflake are missing, for unknown reasons. The
gap is nearly contemporaneous with the gap the measurements for all
transports/bridges caused by BifroestSerge bridge authority switchover,
but starts about 2 weeks earlier in Snowflake's case.",
+ "links" : [ {
+"label" : "ticket",
+"target" : "https://bugs.torproject.org/26783;
+ } ]
+}, {
"start" : "2018-06-27",
"protocols" : [ "meek" ],
"short_description" : "Release of Tor Browser 8.0a9 with non-working meek.",
@@ -3198,6 +3236,9 @@
}, {
"label" : "AllAfrica article",
"target" : "http://allafrica.com/stories/201807040129.html;
+ }, {
+"label" : "OONI report",
+"target" : "https://ooni.io/post/uganda-social-media-tax/;
} ]
}, {
"start" : "2018-07-04",
@@ -3253,6 +3294,25 @@
} ]
}, {
"start"
[tor-commits] [metrics-web/release] Access userstats database from Java only.
commit f5ef5fb0d8f46e28dc7e8536a11d95d43ee61c08
Author: Karsten Loesing
Date: Sat Dec 8 11:06:17 2018 +0100
Access userstats database from Java only.
Previously, we used Java to write .sql files, imported them using
psql, and afterwards made queries via psql. Now we're using Java to
interact with the database directly. This is another step towards
making the daily updater Java-only.
---
build.xml | 22 ---
.../torproject/metrics/stats/clients/Database.java | 156
.../org/torproject/metrics/stats/clients/Main.java | 206 +++--
.../torproject/metrics/stats/clients/Writer.java | 42 +
4 files changed, 263 insertions(+), 163 deletions(-)
diff --git a/build.xml b/build.xml
index 250417e..6736e19 100644
--- a/build.xml
+++ b/build.xml
@@ -367,28 +367,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/src/main/java/org/torproject/metrics/stats/clients/Database.java
b/src/main/java/org/torproject/metrics/stats/clients/Database.java
new file mode 100644
index 000..7e783dc
--- /dev/null
+++ b/src/main/java/org/torproject/metrics/stats/clients/Database.java
@@ -0,0 +1,156 @@
+/* Copyright 2017--2018 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.metrics.stats.clients;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.Timestamp;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.List;
+import java.util.Locale;
+import java.util.TimeZone;
+
+/** Database wrapper to connect to the database, insert data, run the stored
+ * procedure for aggregating data, and query aggregated data as output. */
+class Database implements AutoCloseable {
+
+ /** Database connection string. */
+ private String jdbcString;
+
+ /** Connection object for all interactions with the database. */
+ private Connection connection;
+
+ /** Prepared statement for inserting a platform string into the imported
+ * table. */
+ private PreparedStatement psImportedInsert;
+
+ /** Create a new Database instance and prepare for inserting or querying
+ * data. */
+ Database(String jdbcString) throws SQLException {
+this.jdbcString = jdbcString;
+this.connect();
+this.prepareStatements();
+ }
+
+ private void connect() throws SQLException {
+this.connection = DriverManager.getConnection(this.jdbcString);
+this.connection.setAutoCommit(false);
+ }
+
+ private void prepareStatements() throws SQLException {
+this.psImportedInsert = this.connection.prepareStatement(
+"INSERT INTO imported (fingerprint, node, metric, country, transport, "
++ "version, stats_start, stats_end, val) "
++ "VALUES (?, CAST(? AS node), CAST(? AS metric), ?, ?, ?, ?, ?, ?)");
+ }
+
+ /** Insert into the imported table. */
+ void insertIntoImported(String fingerprint, String node, String metric,
+ String country, String transport, String version, long fromMillis,
+ long toMillis, double val) throws SQLException {
+if (fromMillis > toMillis) {
+ return;
+}
+psImportedInsert.clearParameters();
+psImportedInsert.setString(1, fingerprint);
+psImportedInsert.setString(2, node);
+psImportedInsert.setString(3, metric);
+psImportedInsert.setString(4, country);
+psImportedInsert.setString(5, transport);
+psImportedInsert.setString(6, version);
+psImportedInsert.setTimestamp(7,
+Timestamp.from(Instant.ofEpochMilli(fromMillis)));
+psImportedInsert.setTimestamp(8,
+Timestamp.from(Instant.ofEpochMilli(toMillis)));
+psImportedInsert.setDouble(9, Math.round(val * 10.0) / 10.0);
+psImportedInsert.execute();
+ }
+
+ /** Process the newly imported data by calling the various stored procedures
+ * and then truncating the imported table. */
+ void processImported() throws SQLException {
+this.connection.createStatement().execute("SELECT merge()");
+this.connection.createStatement().execute("SELECT aggregate()");
+this.connection.createStatement().execute("SELECT combine()");
+this.connection.createStatement().execute("TRUNCATE imported");
+ }
+
+ /** Commit all changes made in this execution. */
+ void commit() throws SQLException {
+this.connection.commit();
+ }
+
+ /** Query the estimated view. */
+ List queryEstimated() throws SQLException {
+List statistics = new ArrayList<>();
+String columns = "date, node, country, transport, version, frac, users";
+statistics.add(columns.split(", "));
+Statement st = this.connection.createStatement();
+Cal
[tor-commits] [metrics-web/release] Fix by-country CSV files.
commit 28f567c1ae0458362ac24d491484b81e7bc37afd
Author: Karsten Loesing
Date: Sat Dec 29 10:07:45 2018 +0100
Fix by-country CSV files.
Fixes #28945.
---
src/main/R/rserver/graphs.R | 17 ++---
1 file changed, 10 insertions(+), 7 deletions(-)
diff --git a/src/main/R/rserver/graphs.R b/src/main/R/rserver/graphs.R
index 1ca9357..03b5b93 100644
--- a/src/main/R/rserver/graphs.R
+++ b/src/main/R/rserver/graphs.R
@@ -922,14 +922,15 @@ write_userstats_relay_country <- function(start_p = NULL,
end_p = NULL,
lower = col_double(),
upper = col_double(),
clients = col_double(),
-frac = col_double())) %>%
+frac = col_double()),
+ na = character()) %>%
filter(node == "relay") %>%
filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
filter(if (!is.null(country_p))
country == ifelse(country_p == "all", "", country_p) else TRUE) %>%
-filter(is.na(transport)) %>%
-filter(is.na(version)) %>%
+filter(transport == "") %>%
+filter(version == "") %>%
select(date, country, clients, lower, upper, frac) %>%
rename(users = clients) %>%
write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
@@ -947,14 +948,15 @@ write_userstats_bridge_country <- function(start_p =
NULL, end_p = NULL,
lower = col_double(),
upper = col_double(),
clients = col_double(),
-frac = col_double())) %>%
+frac = col_double()),
+ na = character()) %>%
filter(node == "bridge") %>%
filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
filter(if (!is.null(country_p))
country == ifelse(country_p == "all", "", country_p) else TRUE) %>%
-filter(is.na(transport)) %>%
-filter(is.na(version)) %>%
+filter(transport == "") %>%
+filter(version == "") %>%
select(date, country, clients, frac) %>%
rename(users = clients) %>%
write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
@@ -1031,7 +1033,8 @@ prepare_userstats_bridge_combined <- function(start_p,
end_p, country_p) {
version = col_skip(),
frac = col_double(),
low = col_double(),
-high = col_double())) %>%
+high = col_double()),
+ na = character()) %>%
filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
filter(if (!is.null(country_p)) country == country_p else TRUE)
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Make write_* functions obsolete.
commit 0d2f1e2afd5f4b9e5c533d256586bb03d7466d5f
Author: Karsten Loesing
Date: Thu Jan 10 15:39:04 2019 +0100
Make write_* functions obsolete.
In most cases these functions would call their prepare_* equivalents,
possibly tweak the result, and write it to a .csv file. This patch
moves all those tweaks to the prepare_* functions, possibly reverts
them in the plot_* functions, and makes the write_* functions
obsolete.
The result is not only less code. We're also going to find bugs in
written .csv files sooner, because the same code is now run for
writing graph files, and the latter happens much more often.
---
src/main/R/rserver/graphs.R| 414 +++--
.../torproject/metrics/web/RObjectGenerator.java | 2 +-
2 files changed, 140 insertions(+), 276 deletions(-)
diff --git a/src/main/R/rserver/graphs.R b/src/main/R/rserver/graphs.R
index 27f399d..82a51e7 100644
--- a/src/main/R/rserver/graphs.R
+++ b/src/main/R/rserver/graphs.R
@@ -348,10 +348,17 @@ robust_call <- function(wrappee, filename) {
})
}
+# Write the result of the given FUN, typically a prepare_ function, as .csv
file
+# to the given path_p.
+write_data <- function(FUN, ..., path_p) {
+ FUN(...) %>%
+write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
+}
+
# Disable readr's automatic progress bar.
options(readr.show_progress = FALSE)
-prepare_networksize <- function(start_p, end_p) {
+prepare_networksize <- function(start_p = NULL, end_p = NULL) {
read.csv(paste(stats_dir, "networksize.csv", sep = ""),
colClasses = c("date" = "Date")) %>%
filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
@@ -375,12 +382,7 @@ plot_networksize <- function(start_p, end_p, path_p) {
ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}
-write_networksize <- function(start_p = NULL, end_p = NULL, path_p) {
- prepare_networksize(start_p, end_p) %>%
-write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
-}
-
-prepare_versions <- function(start_p, end_p) {
+prepare_versions <- function(start_p = NULL, end_p = NULL) {
read_csv(paste(stats_dir, "versions.csv", sep = ""),
col_types = cols(
date = col_date(format = ""),
@@ -413,42 +415,34 @@ plot_versions <- function(start_p, end_p, path_p) {
ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}
-write_versions <- function(start_p = NULL, end_p = NULL, path_p) {
- prepare_versions(start_p, end_p) %>%
-write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
-}
-
-prepare_platforms <- function(start_p, end_p) {
+prepare_platforms <- function(start_p = NULL, end_p = NULL) {
read.csv(paste(stats_dir, "platforms.csv", sep = ""),
colClasses = c("date" = "Date")) %>%
filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
-filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE)
+filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
+mutate(platform = tolower(platform)) %>%
+spread(platform, relays)
}
plot_platforms <- function(start_p, end_p, path_p) {
prepare_platforms(start_p, end_p) %>%
+gather(platform, relays, -date) %>%
ggplot(aes(x = date, y = relays, colour = platform)) +
geom_line() +
scale_x_date(name = "", breaks = custom_breaks,
labels = custom_labels, minor_breaks = custom_minor_breaks) +
scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) +
scale_colour_manual(name = "Platform",
- breaks = c("Linux", "macOS", "BSD", "Windows", "Other"),
- values = c("Linux" = "#56B4E9", "macOS" = "#33", "BSD" = "#E69F00",
- "Windows" = "#0072B2", "Other" = "#009E73")) +
+ breaks = c("linux", "macos", "bsd", "windows", "other"),
+ labels = c("Linux", "macOS", "BSD", "Windows", "Other"),
+ values = c("linux" = "#56B4E9", "macos" = "#33", "bsd" = "#E69F00",
+ "windows" = "#0072B2", "other" = "#009E73")) +
ggtitle("Relay platforms") +
labs(caption = copyright_notice)
ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}
-write_platforms <- function(start_p = NULL, end_p = NULL, path_p) {
- prepare_platforms(start_p, end_p) %>%
-mutate(platform = tolower(platform)) %>%
-spread(platform, relays) %>%
-wr
[tor-commits] [metrics-web/release] Tweak Advertised and consumed bandwidth by relay flag graph.
commit 6b5f75996ad0d9ac5151da48f9693d478dd682de
Author: Karsten Loesing
Date: Wed Dec 5 21:32:55 2018 +0100
Tweak Advertised and consumed bandwidth by relay flag graph.
This graph now contains everything that's contained in the Total relay
bandwidth and the Consumed bandwidth by Exit/Guard flag combination
graph.
Removing those graphs will be done in a separate commit.
Part of #28353.
---
src/main/R/rserver/graphs.R | 40 ++--
src/main/resources/web/json/metrics.json | 4 ++--
2 files changed, 20 insertions(+), 24 deletions(-)
diff --git a/src/main/R/rserver/graphs.R b/src/main/R/rserver/graphs.R
index 1f7309b..cb40d52 100644
--- a/src/main/R/rserver/graphs.R
+++ b/src/main/R/rserver/graphs.R
@@ -788,41 +788,37 @@ write_connbidirect <- function(start_p = NULL, end_p =
NULL, path_p) {
prepare_bandwidth_flags <- function(start_p, end_p) {
advbw <- read.csv(paste(stats_dir, "advbw.csv", sep = ""),
colClasses = c("date" = "Date")) %>%
-transmute(date, isguard, isexit, variable = "advbw",
- value = advbw * 8 / 1e9)
+transmute(date, have_guard_flag = isguard, have_exit_flag = isexit,
+ variable = "advbw", value = advbw * 8 / 1e9)
bwhist <- read.csv(paste(stats_dir, "bandwidth.csv", sep = ""),
colClasses = c("date" = "Date")) %>%
-transmute(date, isguard, isexit, variable = "bwhist",
- value = (bwread + bwwrite) * 8 / 2e9)
+transmute(date, have_guard_flag = isguard, have_exit_flag = isexit,
+ variable = "bwhist", value = (bwread + bwwrite) * 8 / 2e9)
rbind(advbw, bwhist) %>%
filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
-group_by(date, variable) %>%
-summarize(exit = sum(value[isexit == "t"]),
- guard = sum(value[isguard == "t"])) %>%
-gather(flag, value, -date, -variable) %>%
-unite(variable, flag, variable) %>%
-mutate(variable = factor(variable,
- levels = c("guard_advbw", "guard_bwhist", "exit_advbw", "exit_bwhist")))
+filter(have_exit_flag != "") %>%
+filter(have_guard_flag != "")
}
plot_bandwidth_flags <- function(start_p, end_p, path_p) {
prepare_bandwidth_flags(start_p, end_p) %>%
-complete(date = full_seq(date, period = 1),
- variable = unique(variable)) %>%
-ggplot(aes(x = date, y = value, colour = variable)) +
-geom_line() +
+unite(flags, have_guard_flag, have_exit_flag) %>%
+mutate(flags = factor(flags, levels = c("f_t", "t_t", "t_f", "f_f"),
+ labels = c("Exit only", "Guard and Exit", "Guard only",
+ "Neither Guard nor Exit"))) %>%
+mutate(variable = ifelse(variable == "advbw",
+ "Advertised bandwidth", "Consumed bandwidth")) %>%
+ggplot(aes(x = date, y = value, fill = flags)) +
+geom_area() +
scale_x_date(name = "", breaks = custom_breaks,
labels = custom_labels, minor_breaks = custom_minor_breaks) +
scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"),
limits = c(0, NA)) +
-scale_colour_manual(name = "",
-breaks = c("guard_advbw", "guard_bwhist", "exit_advbw", "exit_bwhist"),
-labels = c("Guard, advertised bandwidth", "Guard, bandwidth history",
- "Exit, advertised bandwidth", "Exit, bandwidth history"),
-values = c("#E69F00", "#D6C827", "#009E73", "#00C34F")) +
-ggtitle(paste("Advertised bandwidth and bandwidth history by",
-"relay flags")) +
+scale_fill_manual(name = "",
+ values = c("#03B3FF", "#39FF02", "#00", "#99")) +
+facet_grid(variable ~ .) +
+ggtitle("Advertised and consumed bandwidth by relay flags") +
labs(caption = copyright_notice) +
theme(legend.position = "top")
ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
diff --git a/src/main/resources/web/json/metrics.json
b/src/main/resources/web/json/metrics.json
index b351814..5173ae4 100644
--- a/src/main/resources/web/json/metrics.json
+++ b/src/main/resources/web/json/metrics.json
@@ -90,9 +90,9 @@
},
{
"id": "bandwidth-flags",
-"title": "Advertised and consumed bandwidth by relay flag",
+"title": "Advertised and consumed bandwidth by relay flags",
"type&
[tor-commits] [metrics-web/release] Split up huge plot_userstats function.
commit f55e63d986ed9c1054ce19ff0d4a19b1c0bce26d
Author: Karsten Loesing
Date: Thu Jan 10 09:54:39 2019 +0100
Split up huge plot_userstats function.
The mere size of this function made it hard to impossible to refactor
things to using more recent R packages dplyr and tidyr. Now there are
four plot_userstats_* functions with accompanying prepare_userstats_*
that make the corresponding write_userstats_* functions really small.
---
src/main/R/rserver/graphs.R | 269 +++-
1 file changed, 115 insertions(+), 154 deletions(-)
diff --git a/src/main/R/rserver/graphs.R b/src/main/R/rserver/graphs.R
index d3ea90a..ba8862c 100644
--- a/src/main/R/rserver/graphs.R
+++ b/src/main/R/rserver/graphs.R
@@ -751,9 +751,9 @@ write_bandwidth_flags <- function(start_p = NULL, end_p =
NULL, path_p) {
write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}
-plot_userstats <- function(start_p, end_p, node_p, variable_p, value_p,
-events_p, path_p) {
- c <- read_csv(file = paste(stats_dir, "clients.csv", sep = ""),
+prepare_userstats_relay_country <- function(start_p, end_p, country_p,
+events_p) {
+ read_csv(file = paste(stats_dir, "clients.csv", sep = ""),
col_types = cols(
date = col_date(format = ""),
node = col_character(),
@@ -763,97 +763,26 @@ plot_userstats <- function(start_p, end_p, node_p,
variable_p, value_p,
lower = col_double(),
upper = col_double(),
clients = col_double(),
-frac = col_skip()),
+frac = col_double()),
na = character()) %>%
-filter(node == node_p)
- u <- c[c$date >= start_p & c$date <= end_p, c("date", "country", "transport",
- "version", "lower", "upper", "clients")]
- u <- rbind(u, data.frame(date = start_p,
- country = ifelse(variable_p == "country" & value_p != "all", value_p,
""),
- transport = ifelse(variable_p == "transport", value_p, ""),
- version = ifelse(variable_p == "version", value_p, ""),
- lower = 0, upper = 0, clients = 0))
- if (node_p == "relay") {
-if (value_p != "all") {
- u <- u[u$country == value_p, ]
- title <- paste("Directly connecting users from", countryname(value_p))
-} else {
- u <- u[u$country == "", ]
- title <- "Directly connecting users"
-}
-u <- aggregate(list(lower = u$lower, upper = u$upper,
-users = u$clients),
- by = list(date = as.Date(u$date, "%Y-%m-%d"),
- value = u$country),
- FUN = sum)
- } else if (variable_p == "transport") {
-if ("!" %in% value_p) {
- n <- u[u$transport != "" & u$transport != "", ]
- n <- aggregate(list(lower = n$lower, upper = n$upper,
- clients = n$clients),
- by = list(date = n$date),
- FUN = sum)
- u <- rbind(u, data.frame(date = n$date,
- country = "", transport = "!",
- version = "", lower = n$lower,
- upper = n$upper, clients = n$clients))
-}
-if (length(value_p) > 1) {
- u <- u[u$transport %in% value_p, ]
- u <- aggregate(list(lower = u$lower, upper = u$upper,
- users = u$clients),
- by = list(date = as.Date(u$date, "%Y-%m-%d"),
- value = u$transport),
- FUN = sum)
- title <- paste("Bridge users by transport")
-} else {
- u <- u[u$transport == value_p, ]
- u <- aggregate(list(lower = u$lower, upper = u$upper,
- users = u$clients),
- by = list(date = as.Date(u$date, "%Y-%m-%d"),
- value = u$transport),
- FUN = sum)
- title <- paste("Bridge users using",
- ifelse(value_p == "", "unknown pluggable transport(s)",
- ifelse(value_p == "", "default OR protocol",
- ifelse(value_p == "!", "any pluggable transport",
- ifelse(value_p == "fte", "FTE",
- ifelse(value_p == "websocket", "Flash proxy/websocket",
- paste("transport", value_p)))
-}
- } else if (variable_p == "version") {
-u <- u[u$version == value_p, ]
-title <- paste("Bridge users
[tor-commits] [metrics-web/release] Fix Traffic link on start page.
commit 87f922d4fd555804d4e80fdefd7968acce5f4433 Author: Karsten Loesing Date: Sat Dec 29 09:23:35 2018 +0100 Fix Traffic link on start page. --- src/main/resources/web/jsps/index.jsp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/web/jsps/index.jsp b/src/main/resources/web/jsps/index.jsp index 3fa49b8..ec93792 100644 --- a/src/main/resources/web/jsps/index.jsp +++ b/src/main/resources/web/jsps/index.jsp @@ -42,7 +42,7 @@ - Traffic How much traffic the Tor network can handle and how much traffic there is. + Traffic How much traffic the Tor network can handle and how much traffic there is. ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Remove two unused R files from censorship detector.
commit c0a18aab9092c57f107732cb2f97f034909e94d9
Author: Karsten Loesing
Date: Thu Dec 20 14:09:24 2018 +0100
Remove two unused R files from censorship detector.
Still part of #21588.
---
src/main/R/clients/merge-clients.R | 19 ---
src/main/R/clients/userstats-detector.R | 18 --
2 files changed, 37 deletions(-)
diff --git a/src/main/R/clients/merge-clients.R
b/src/main/R/clients/merge-clients.R
deleted file mode 100644
index cce7e9d..000
--- a/src/main/R/clients/merge-clients.R
+++ /dev/null
@@ -1,19 +0,0 @@
-require(reshape)
-r <- read.csv("userstats-ranges.csv", stringsAsFactors = FALSE)
-r <- melt(r, id.vars = c("date", "country"))
-r <- data.frame(date = r$date, node = "relay", country = r$country,
- transport = "", version = "",
- variable = ifelse(r$variable == "maxusers", "upper", "lower"),
- value = floor(r$value))
-u <- read.csv("userstats.csv", stringsAsFactors = FALSE)
-u <- melt(u, id.vars = c("date", "node", "country", "transport",
- "version"))
-u <- data.frame(date = u$date, node = u$node, country = u$country,
- transport = u$transport, version = u$version,
- variable = ifelse(u$variable == "frac", "frac", "clients"),
- value = u$value)
-c <- rbind(r, u)
-c <- cast(c, date + node + country + transport + version ~ variable)
-c <- c[order(as.Date(c$date), c$node, c$country, c$transport, c$version), ]
-write.csv(c, "clients.csv", quote = FALSE, row.names = FALSE, na = "")
-
diff --git a/src/main/R/clients/userstats-detector.R
b/src/main/R/clients/userstats-detector.R
deleted file mode 100644
index c3a9041..000
--- a/src/main/R/clients/userstats-detector.R
+++ /dev/null
@@ -1,18 +0,0 @@
-library("reshape")
-export_userstats_detector <- function(path) {
- c <- read.csv("userstats.csv", stringsAsFactors = FALSE)
- c <- c[c$country != '' & c$transport == '' & c$version == '' &
- c$node == 'relay', ]
- u <- data.frame(country = c$country, date = c$date, users = c$users,
- stringsAsFactors = FALSE)
- u <- rbind(u, data.frame(country = "zy",
-aggregate(list(users = u$users),
- by = list(date = u$date), sum)))
- u <- data.frame(date = u$date, country = u$country,
- users = floor(u$users))
- u <- cast(u, date ~ country, value = "users")
- names(u)[names(u) == "zy"] <- "all"
- write.csv(u, path, quote = FALSE, row.names = FALSE)
-}
-export_userstats_detector("userstats-detector.csv")
-
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Take out "We're hiring" link from start page.
commit 54930cb95f685f7a42610edd4ae2a39f2add7116 Author: Karsten Loesing Date: Wed Dec 19 21:05:01 2018 +0100 Take out "We're hiring" link from start page. --- src/main/resources/web/jsps/index.jsp | 5 - 1 file changed, 5 deletions(-) diff --git a/src/main/resources/web/jsps/index.jsp b/src/main/resources/web/jsps/index.jsp index fbccd7d..3fa49b8 100644 --- a/src/main/resources/web/jsps/index.jsp +++ b/src/main/resources/web/jsps/index.jsp @@ -6,11 +6,6 @@ - - -We're hiring The Tor Project is seeking an experienced Data Architect to take our metrics work to the next level. https://www.torproject.org/about/jobs-metrics-data-architect.html.en; target="_blank">Read the job posting. - - Welcome to Tor Metrics! ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Remove unused parts of totalcw module.
commit e94ceeb9b6763374f6edb0105ae6da9fd5098d99
Author: Karsten Loesing
Date: Thu Nov 29 08:07:11 2018 +0100
Remove unused parts of totalcw module.
Requires updating the vote table of the database.
Part of #28137, #28328, and #28352.
---
.../torproject/metrics/stats/totalcw/Database.java | 15 ++--
.../torproject/metrics/stats/totalcw/Parser.java | 36 +--
.../totalcw/TotalcwRelayNetworkStatusVote.java | 25 -
src/main/sql/totalcw/init-totalcw.sql | 23
.../totalcw/TotalcwRelayNetworkStatusVoteTest.java | 42 +++---
5 files changed, 17 insertions(+), 124 deletions(-)
diff --git a/src/main/java/org/torproject/metrics/stats/totalcw/Database.java
b/src/main/java/org/torproject/metrics/stats/totalcw/Database.java
index be4cad3..66b0366 100644
--- a/src/main/java/org/torproject/metrics/stats/totalcw/Database.java
+++ b/src/main/java/org/torproject/metrics/stats/totalcw/Database.java
@@ -66,10 +66,8 @@ class Database implements AutoCloseable {
"SELECT EXISTS (SELECT 1 FROM vote "
+ "WHERE valid_after = ? AND authority_id = ?)");
this.psVoteInsert = this.connection.prepareStatement(
-"INSERT INTO vote (valid_after, authority_id, measured_count, "
-+ "measured_sum, measured_mean, measured_min, measured_q1, "
-+ "measured_median, measured_q3, measured_max) "
-+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+"INSERT INTO vote (valid_after, authority_id, measured_sum) "
++ "VALUES (?, ?, ?)",
Statement.RETURN_GENERATED_KEYS);
}
@@ -124,14 +122,7 @@ class Database implements AutoCloseable {
Timestamp.from(ZonedDateTime.of(vote.validAfter,
ZoneId.of("UTC")).toInstant()), calendar);
this.psVoteInsert.setInt(2, authorityId);
-this.psVoteInsert.setLong(3, vote.measuredCount);
-this.psVoteInsert.setLong(4, vote.measuredSum);
-this.psVoteInsert.setLong(5, vote.measuredMean);
-this.psVoteInsert.setLong(6, vote.measuredMin);
-this.psVoteInsert.setLong(7, vote.measuredQ1);
-this.psVoteInsert.setLong(8, vote.measuredMedian);
-this.psVoteInsert.setLong(9, vote.measuredQ3);
-this.psVoteInsert.setLong(10, vote.measuredMax);
+this.psVoteInsert.setLong(3, vote.measuredSum);
this.psVoteInsert.execute();
try (ResultSet rs = this.psVoteInsert.getGeneratedKeys()) {
if (rs.next()) {
diff --git a/src/main/java/org/torproject/metrics/stats/totalcw/Parser.java
b/src/main/java/org/torproject/metrics/stats/totalcw/Parser.java
index 4367200..893184c 100644
--- a/src/main/java/org/torproject/metrics/stats/totalcw/Parser.java
+++ b/src/main/java/org/torproject/metrics/stats/totalcw/Parser.java
@@ -6,13 +6,8 @@ package org.torproject.metrics.stats.totalcw;
import org.torproject.descriptor.NetworkStatusEntry;
import org.torproject.descriptor.RelayNetworkStatusVote;
-import org.apache.commons.math3.stat.descriptive.rank.Percentile;
-
import java.time.Instant;
import java.time.ZoneId;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
/** Parser that extracts bandwidth measurement statistics from votes and
creates
* data objects for them. */
@@ -22,13 +17,17 @@ class Parser {
* contain any bandwidth measurements. */
TotalcwRelayNetworkStatusVote parseRelayNetworkStatusVote(
RelayNetworkStatusVote vote) {
-List measuredBandwidths = new ArrayList<>();
+Long measuredSum = null;
for (NetworkStatusEntry entry : vote.getStatusEntries().values()) {
- if (entry.getMeasured() >= 0L) {
-measuredBandwidths.add(entry.getMeasured());
+ if (entry.getMeasured() < 0L) {
+continue;
+ }
+ if (null == measuredSum) {
+measuredSum = 0L;
}
+ measuredSum += entry.getMeasured();
}
-if (measuredBandwidths.isEmpty()) {
+if (null == measuredSum) {
/* Return null, because we wouldn't want to add this vote to the database
* anyway. */
return null;
@@ -39,24 +38,7 @@ class Parser {
.atZone(ZoneId.of("UTC")).toLocalDateTime();
parsedVote.identityHex = vote.getIdentity();
parsedVote.nickname = vote.getNickname();
-Collections.sort(measuredBandwidths);
-long totalValue = 0L;
-double[] values = new double[measuredBandwidths.size()];
-for (int i = 0; i < measuredBandwidths.size(); i++) {
- values[i] = (double) measuredBandwidths.get(i);
- totalValue += measuredBandwidths.get(i);
-}
-parsedVote.measuredCount = values.length;
-parsedVote.measuredSum = totalValue;
-parsedVote.measuredMean = totalValue / values.length;
-parsedVote.measuredMin = (long) Math.floor(values[0]);
-parsedVote.measuredMax = (long) Math.floor(values[values.length - 1]);
-Percentile
[tor-commits] [metrics-web/release] Rewrite censorship detector in Java.
commit a367168a782e864bdacb610857b1dc5d58fd192d
Author: Karsten Loesing
Date: Sun Dec 9 12:02:42 2018 +0100
Rewrite censorship detector in Java.
This allows us to remove the last remaining Python parts from the daily
updater.
Implements #21588.
---
build.xml | 26 --
.../torproject/metrics/stats/clients/Detector.java | 433 +
.../org/torproject/metrics/stats/clients/Main.java | 5 +
src/main/python/clients/country_info.py| 255
src/main/python/clients/detector.py| 242
5 files changed, 438 insertions(+), 523 deletions(-)
diff --git a/build.xml b/build.xml
index 6736e19..93eda7b 100644
--- a/build.xml
+++ b/build.xml
@@ -23,7 +23,6 @@
-
@@ -360,32 +359,7 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/src/main/java/org/torproject/metrics/stats/clients/Detector.java
b/src/main/java/org/torproject/metrics/stats/clients/Detector.java
new file mode 100644
index 000..1a523c2
--- /dev/null
+++ b/src/main/java/org/torproject/metrics/stats/clients/Detector.java
@@ -0,0 +1,433 @@
+/* Copyright 2011 George Danezis
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted (subject to the limitations in the
+ * disclaimer below) provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ *notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ *notice, this list of conditions and the following disclaimer in the
+ *documentation and/or other materials provided with the
+ *distribution.
+ *
+ * * Neither the name of nor the names of its
+ *contributors may be used to endorse or promote products derived
+ *from this software without specific prior written permission.
+ *
+ * NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE
+ * GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT
+ * HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * (Clear BSD license:
+ * http://labs.metacarta.com/license-explanation.html#license)
+ *
+ * Copyright 2018 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.metrics.stats.clients;
+
+import org.apache.commons.math3.distribution.NormalDistribution;
+import org.apache.commons.math3.distribution.PoissonDistribution;
+import org.apache.commons.math3.stat.descriptive.moment.Mean;
+import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation;
+import org.apache.commons.math3.stat.descriptive.rank.Percentile;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.LineNumberReader;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.LocalDate;
+import java.time.format.DateTimeParseException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
+
+/** Censorship detector that reads a .csv file of the number of Tor clients and
+ * finds anomalies that might be indicative of censorship. */
+public class Detector {
+
+ /** Input file. */
+ private static final Path INPUT_PATH = Paths.get("stats", "userstats.csv");
+
+ /** Output file. */
+ private static final Path OUTPUT_PATH = Paths.get("stats", "clients.csv");
+
+ /** Number of largest locations to be included in the detection algorithm. */
+ private static final int NUM_LARGEST_LOCATIONS = 50;
+
+ /** Time interval in days to model connection rates. */
+ private static final int INTERV = 7;
+
+ /** Compound key under which client estimates are stored in both input and
+ * output files. */
+ private static class ClientsKey implements Comparable {
+
+/** Date when clients connected
[tor-commits] [metrics-web/release] Extend ipv6servers to replace servers.csv.
commit 54976512ff47cc3b8a2c6b5ab2cecc98b1ba8088
Author: Karsten Loesing
Date: Mon Oct 29 19:41:12 2018 +0100
Extend ipv6servers to replace servers.csv.
This change extends the ipv6servers module to generate all relevant
data about servers (relays and bridges) that the legacy module
currently generates.
There are several reasons why this is useful:
- This change is a step towards implementing all statistics in Java,
without needing to call psql or other external tools from within
Ant. In fact, it's a step towards getting rid of Ant for executing
modules.
- The ipv6servers module already supports other statistics than
absolute server counts, namely advertised bandwidths. It's easy to
extend statistics to consensus weights and guard/middle/exit
probabilities. This prepares future graphs which can be added
quite easily.
- With these new statistics we can finally provide graphs on bridges
by version or platform. Or we can extend current graphs to display
both relays and bridges, if we want to avoid adding more graphs.
This commit does not yet remove any code from the legacy module. That
will be the next logical step. It will even be fun.
Implements #28116.
---
CHANGELOG.md | 2 +
src/main/R/rserver/graphs.R| 45 +--
.../metrics/stats/ipv6servers/Configuration.java | 2 +-
.../metrics/stats/ipv6servers/Database.java| 416 ++---
.../stats/ipv6servers/Ipv6NetworkStatus.java | 48 ++-
.../stats/ipv6servers/Ipv6ServerDescriptor.java| 8 +
.../torproject/metrics/stats/ipv6servers/Main.java | 24 +-
.../metrics/stats/ipv6servers/OutputLine.java | 72
.../metrics/stats/ipv6servers/Parser.java | 158 ++--
.../metrics/stats/ipv6servers/Writer.java | 14 +-
src/main/sql/ipv6servers/init-ipv6servers.sql | 410 ++--
11 files changed, 972 insertions(+), 227 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f604ff9..028df01 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,8 @@
* Medium changes
- Start downloading and processing votes.
- Add Apache Commons Math 3.6.1 as dependency.
+ - Extend ipv6servers module to generate servers part of legacy
+ module.
# Changes in version 1.2.0 - 2018-08-25
diff --git a/src/main/R/rserver/graphs.R b/src/main/R/rserver/graphs.R
index b021791..9dc8c2d 100644
--- a/src/main/R/rserver/graphs.R
+++ b/src/main/R/rserver/graphs.R
@@ -349,16 +349,10 @@ robust_call <- function(wrappee, filename) {
}
prepare_networksize <- function(start_p, end_p) {
- read.csv(paste(stats_dir, "servers.csv", sep = ""),
+ read.csv(paste(stats_dir, "networksize.csv", sep = ""),
colClasses = c("date" = "Date")) %>%
filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
-filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
-filter(flag == "") %>%
-filter(country == "") %>%
-filter(version == "") %>%
-filter(platform == "") %>%
-filter(ec2bridge == "") %>%
-select(date, relays, bridges)
+filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE)
}
plot_networksize <- function(start_p, end_p, path_p) {
@@ -384,16 +378,10 @@ write_networksize <- function(start_p = NULL, end_p =
NULL, path_p) {
}
prepare_versions <- function(start_p, end_p) {
- read.csv(paste(stats_dir, "servers.csv", sep = ""),
+ read.csv(paste(stats_dir, "versions.csv", sep = ""),
colClasses = c("date" = "Date")) %>%
filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
-filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
-filter(flag == "") %>%
-filter(country == "") %>%
-filter(version != "") %>%
-filter(platform == "") %>%
-filter(ec2bridge == "") %>%
-select(date, version, relays)
+filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE)
}
plot_versions <- function(start_p, end_p, path_p) {
@@ -428,18 +416,10 @@ write_versions <- function(start_p = NULL, end_p = NULL,
path_p) {
}
prepare_platforms <- function(start_p, end_p) {
- read.csv(paste(stats_dir, "servers.csv", sep = ""),
+ read.csv(paste(stats_dir, "platforms.csv", sep = ""),
colClasses = c("date" = "Date")) %>%
filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
-filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
-filter(flag == "") %>%
-
[tor-commits] [metrics-web/release] Remove Torperf/OnionPerf plots with all sources.
commit c39472548511175a6eaa0d67de62d3b5fa59dbe3
Author: Karsten Loesing
Date: Wed Dec 5 11:56:19 2018 +0100
Remove Torperf/OnionPerf plots with all sources.
OnionPerf results look to be comparable over time, but between vantage
points there are systematic deltas between the results. The "all"
plots show rises and falls where they actually don't exist, it's just
that a particular vantage point was offline so the average of the two
remaining moves noticeably.
In this commit we remove the source parameter from these graphs and
always include all sources separately in the graph, but not a
combination of all measurements together.
Implements #28603.
---
src/main/R/rserver/graphs.R| 178 +
.../metrics/web/GraphParameterChecker.java | 24 ---
.../org/torproject/metrics/web/GraphServlet.java | 8 -
src/main/resources/web/json/metrics.json | 7 +-
src/main/resources/web/jsps/graph.jsp | 9 --
5 files changed, 76 insertions(+), 150 deletions(-)
diff --git a/src/main/R/rserver/graphs.R b/src/main/R/rserver/graphs.R
index e541c30..1f7309b 100644
--- a/src/main/R/rserver/graphs.R
+++ b/src/main/R/rserver/graphs.R
@@ -592,70 +592,49 @@ write_relayflags <- function(start_p = NULL, end_p =
NULL, flag_p = NULL,
write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}
-plot_torperf <- function(start_p, end_p, source_p, server_p, filesize_p,
-path_p) {
- filesize_val <- ifelse(filesize_p == "50kb", 50 * 1024,
- ifelse(filesize_p == "1mb", 1024 * 1024, 5 * 1024 * 1024))
- t <- read.csv(paste(stats_dir, "torperf-1.1.csv", sep = ""),
-colClasses = c("date" = "Date", "source" = "character"))
- known_sources <- c("all", unique(t[t$source != "", "source"]))
- colours <- data.frame(source = known_sources,
- colour = brewer.pal(length(known_sources), "Paired"),
- stringsAsFactors = FALSE)
- colour <- colours[colours$source == source_p, "colour"]
- filesizes <- data.frame(filesizes = c("5mb", "1mb", "50kb"),
- label = c("5 MiB", "1 MiB", "50 KiB"), stringsAsFactors = FALSE)
- filesize_str <- filesizes[filesizes$filesize == filesize_p, "label"]
- t[t$date >= as.Date(start_p) & t$date <= as.Date(end_p) &
- t$filesize == filesize_val &
- t$source == ifelse(source_p == "all", "", source_p) &
- t$server == server_p, ] %>%
-transmute(date, q1 = q1 / 1e3, md = md / 1e3, q3 = q3 / 1e3) %>%
-complete(date = full_seq(date, period = 1)) %>%
-ggplot(aes(x = date, y = md, fill = "line")) +
-geom_line(colour = colour, size = 0.75) +
-geom_ribbon(aes(x = date, ymin = q1, ymax = q3, fill = "ribbon")) +
+prepare_torperf <- function(start_p, end_p, server_p, filesize_p, path_p) {
+ read.csv(paste(stats_dir, "torperf-1.1.csv", sep = ""),
+colClasses = c("date" = "Date", "source" = "character")) %>%
+filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
+filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
+filter(if (!is.null(server_p)) server == server_p else TRUE) %>%
+filter(if (!is.null(filesize_p))
+filesize == ifelse(filesize_p == "50kb", 50 * 1024,
+ifelse(filesize_p == "1mb", 1024 * 1024, 5 * 1024 * 1024)) else
+TRUE) %>%
+transmute(date, filesize, source, server, q1 = q1 / 1e3, md = md / 1e3,
+ q3 = q3 / 1e3)
+}
+
+plot_torperf <- function(start_p, end_p, server_p, filesize_p, path_p) {
+ prepare_torperf(start_p, end_p, server_p, filesize_p, path_p) %>%
+filter(source != "") %>%
+complete(date = full_seq(date, period = 1), nesting(source)) %>%
+ggplot(aes(x = date, y = md, ymin = q1, ymax = q3, fill = source)) +
+geom_ribbon(alpha = 0.5) +
+geom_line(aes(colour = source), size = 0.75) +
scale_x_date(name = "", breaks = custom_breaks,
labels = custom_labels, minor_breaks = custom_minor_breaks) +
scale_y_continuous(name = "", labels = unit_format(unit = "s"),
limits = c(0, NA)) +
-scale_fill_manual(name = paste("Measured times on",
-ifelse(source_p == "all", "all sources", source_p), "per day"),
- breaks = c("line", "ribbon"),
- labels = c("Median", "1st to 3rd quartile"),
- values = paste(colour, c("", "66"), sep = "")) +
-ggtitle(paste("Time to complete", filesize_
[tor-commits] [metrics-web/release] Remove bandwidth and bwhist-flags graphs.
commit 59b800a491f93d08bcec6bda1b393a762156226d
Author: Karsten Loesing
Date: Wed Dec 12 12:08:52 2018 +0100
Remove bandwidth and bwhist-flags graphs.
The "Advertised and consumed bandwidth by relay flags" graph now
contains everything that's contained in the "Total relay bandwidth"
and the "Consumed bandwidth by Exit/Guard flag combination" graphs.
Removing these two graphs as obsolete.
Also update documentation for the newly deployed "Advertised and
consumed bandwidth by relay flags" graph.
Part of #28353.
---
src/main/R/rserver/graphs.R| 79 --
src/main/resources/web.xml | 8 ---
src/main/resources/web/json/categories.json| 2 -
src/main/resources/web/json/metrics.json | 22 --
.../resources/web/jsps/reproducible-metrics.jsp| 17 ++---
src/main/resources/web/jsps/stats.jsp | 68 ++-
6 files changed, 13 insertions(+), 183 deletions(-)
diff --git a/src/main/R/rserver/graphs.R b/src/main/R/rserver/graphs.R
index cb40d52..1ca9357 100644
--- a/src/main/R/rserver/graphs.R
+++ b/src/main/R/rserver/graphs.R
@@ -448,85 +448,6 @@ write_platforms <- function(start_p = NULL, end_p = NULL,
path_p) {
write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}
-prepare_bandwidth <- function(start_p, end_p) {
- advbw <- read.csv(paste(stats_dir, "advbw.csv", sep = ""),
-colClasses = c("date" = "Date")) %>%
-transmute(date, variable = "advbw", value = advbw * 8 / 1e9)
- bwhist <- read.csv(paste(stats_dir, "bandwidth.csv", sep = ""),
-colClasses = c("date" = "Date")) %>%
-transmute(date, variable = "bwhist", value = (bwread + bwwrite) * 8 / 2e9)
- rbind(advbw, bwhist) %>%
-filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
-filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
-filter(!is.na(value)) %>%
-group_by(date, variable) %>%
-summarize(value = sum(value)) %>%
-spread(variable, value)
-}
-
-plot_bandwidth <- function(start_p, end_p, path_p) {
- prepare_bandwidth(start_p, end_p) %>%
-gather(variable, value, -date) %>%
-ggplot(aes(x = date, y = value, colour = variable)) +
-geom_line() +
-scale_x_date(name = "", breaks = custom_breaks,
- labels = custom_labels, minor_breaks = custom_minor_breaks) +
-scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"),
- limits = c(0, NA)) +
-scale_colour_hue(name = "", h.start = 90,
-breaks = c("advbw", "bwhist"),
-labels = c("Advertised bandwidth", "Bandwidth history")) +
-ggtitle("Total relay bandwidth") +
-labs(caption = copyright_notice) +
-theme(legend.position = "top")
- ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
-}
-
-write_bandwidth <- function(start_p = NULL, end_p = NULL, path_p) {
- prepare_bandwidth(start_p, end_p) %>%
-write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
-}
-
-prepare_bwhist_flags <- function(start_p, end_p) {
- read.csv(paste(stats_dir, "bandwidth.csv", sep = ""),
-colClasses = c("date" = "Date")) %>%
-filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
-filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
-filter(isexit != "") %>%
-filter(isguard != "") %>%
-mutate(variable = ifelse(isexit == "t",
-ifelse(isguard == "t", "guard_and_exit", "exit_only"),
-ifelse(isguard == "t", "guard_only", "middle_only")),
- value = (bwread + bwwrite) * 8 / 2e9) %>%
-select(date, variable, value)
-}
-
-plot_bwhist_flags <- function(start_p, end_p, path_p) {
- prepare_bwhist_flags(start_p, end_p) %>%
-complete(date = full_seq(date, period = 1),
- variable = unique(variable)) %>%
-ggplot(aes(x = date, y = value, colour = variable)) +
-geom_line() +
-scale_x_date(name = "", breaks = custom_breaks,
- labels = custom_labels, minor_breaks = custom_minor_breaks) +
-scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"),
- limits = c(0, NA)) +
-scale_colour_manual(name = "",
- breaks = c("exit_only", "guard_and_exit", "guard_only", "middle_only"),
- labels = c("Exit only", "Guard & Exit", "Guard only", "Middle only"),
- values = c("#E69F00", "#56B
[tor-commits] [metrics-web/release] Document changes to the totalcw graph.
commit ffaab885748c5340b01ad87ddb88819cd779c2b0 Author: Karsten Loesing Date: Thu Nov 29 11:53:41 2018 +0100 Document changes to the totalcw graph. Still part of #28137, #28328, and #28352. --- .../resources/web/jsps/reproducible-metrics.jsp| 22 ++ src/main/resources/web/jsps/stats.jsp | 6 +++--- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/main/resources/web/jsps/reproducible-metrics.jsp b/src/main/resources/web/jsps/reproducible-metrics.jsp index b6df6c3..24bdba0 100644 --- a/src/main/resources/web/jsps/reproducible-metrics.jsp +++ b/src/main/resources/web/jsps/reproducible-metrics.jsp @@ -380,7 +380,18 @@ The goal is to avoid over-representing a few statuses during periods when the br Total consensus weights across bandwidth authorities graph -Step 1: Parse votes. +Step 1: Parse consensuses. + +Obtain consensuses from CollecTor. +Refer to the https://gitweb.torproject.org/torspec.git/tree/dir-spec.txt;>Tor directory protocol, version 3 for details on the descriptor format. + +Parse and memorize the "valid-after" time from the consensus header. We use this UTC timestamp to aggregate by the UTC date. + +Parse the "s" lines of all status entries and skip entries without the "Running" flag. Optionally distinguish relays by assigned "Guard" and "Exit" flags. + +Parse the (optional) "w" lines of all status entries and compute the total of all bandwidth values denoted by the "Bandwidth=" keyword. If an entry does not contain such a value, skip the entry. If a consensus does not contain a single bandwidth value, skip the consensus. + +Step 2: Parse votes. Obtain votes from CollecTor. Refer to the https://gitweb.torproject.org/torspec.git/tree/dir-spec.txt;>Tor directory protocol, version 3 for details on the descriptor format. @@ -389,11 +400,14 @@ Refer to the https://gitweb.torproject.org/torspec.git/tree/dir-spec.tx Also parse the "nickname" and "identity" fields from the "dir-source" line. We use the identity to aggregate by authority and the nickname for display purposes. -Parse the (optional) "w" lines of all status entries and compute the total of all measured bandwidth values denoted by the "Measured=" keyword. If an entry does not contain such a value, skip the entry. If a vote does not contain a single measured bandwidth value, skip the vote. +Parse the "s" lines of all status entries and skip entries without the "Running" flag. Optionally distinguish relays by assigned "Guard" and "Exit" flags. -Step 2: Compute daily averages +Parse the (optional) "w" lines of all status entries and compute the total of all measured bandwidth values denoted by the "Measured=" keyword. If an entry does not contain such a value, skip the entry. If a vote does not contain a single measured bandwidth value, skip the vote. + +Step 3: Compute daily averages -Go through all previously processed votes by valid-after UTC date and authority. +Go through all previously processed consensuses and votes by valid-after UTC date and authority. +If there are less than 12 consensuses known for a given UTC date, skip consensuses from this date. If an authority published less than 12 votes on a given UTC date, skip this date and authority. Also skip the last date of the results, because those averages may still change throughout the day. For all remaining combinations of date and authority, compute the arithmetic mean of total measured bandwidth, rounded down to the next-smaller integer number. diff --git a/src/main/resources/web/jsps/stats.jsp b/src/main/resources/web/jsps/stats.jsp index 002a3af..e5f9c6a 100644 --- a/src/main/resources/web/jsps/stats.jsp +++ b/src/main/resources/web/jsps/stats.jsp @@ -321,9 +321,9 @@ Servers # Columns -date: UTC date (-MM-DD) when bridges have been listed as running. -nickname: Bandwidth authority nickname. -totalcw: Total consensus weight of all relays measured by the bandwidth authority. +date: UTC date (-MM-DD) when relays have been listed as running. +nickname: Bandwidth authority nickname, or the empty string in case of the consensus. +totalcw: Total consensus weight of all running relays measured by the bandwidth authority or contained in the consensus. ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Modernize legacy module and rename it to bwhist.
commit f8fa108d183968540eca529250cb142f8216ce8c
Author: Karsten Loesing
Date: Wed Nov 14 10:39:24 2018 +0100
Modernize legacy module and rename it to bwhist.
Changes include using similar mechanisms for configuration, calling
the database aggregation function, querying the database, and writing
results as we're using in the ipv6servers and other modules.
Configuration options can now be changed via the following Java
properties:
bwhist.descriptors
bwhist.database
bwhist.history
bwhist.output
The legacy.config file, if one exists, will be ignored.
Part of #28116.
---
build.xml | 38 +-
.../metrics/stats/bwhist/Configuration.java| 18 +++
.../org/torproject/metrics/stats/bwhist/Main.java | 56 +
.../RelayDescriptorDatabaseImporter.java | 131 +++--
.../torproject/metrics/stats/bwhist/Writer.java| 42 +++
.../metrics/stats/servers/Configuration.java | 87 --
.../org/torproject/metrics/stats/servers/Main.java | 40 ---
src/main/resources/legacy.config.template | 8 --
8 files changed, 212 insertions(+), 208 deletions(-)
diff --git a/build.xml b/build.xml
index b95550d..a391416 100644
--- a/build.xml
+++ b/build.xml
@@ -315,7 +315,7 @@
-
+
@@ -340,39 +340,9 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
-
-
-
-
-
-
-
-
-
-
-
@@ -503,7 +473,7 @@
-
+
diff --git
a/src/main/java/org/torproject/metrics/stats/bwhist/Configuration.java
b/src/main/java/org/torproject/metrics/stats/bwhist/Configuration.java
new file mode 100644
index 000..2a0fbc5
--- /dev/null
+++ b/src/main/java/org/torproject/metrics/stats/bwhist/Configuration.java
@@ -0,0 +1,18 @@
+/* Copyright 2011--2018 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.metrics.stats.bwhist;
+
+/** Configuration options parsed from Java properties with reasonable
hard-coded
+ * defaults. */
+public class Configuration {
+ static String descriptors = System.getProperty("bwhist.descriptors",
+ "../../shared/in/");
+ static String database = System.getProperty("bwhist.database",
+ "jdbc:postgresql:tordir");
+ static String history = System.getProperty("bwhist.history",
+ "status/read-descriptors");
+ static String output = System.getProperty("bwhist.output",
+ "stats/");
+}
+
diff --git a/src/main/java/org/torproject/metrics/stats/bwhist/Main.java
b/src/main/java/org/torproject/metrics/stats/bwhist/Main.java
new file mode 100644
index 000..61c1435
--- /dev/null
+++ b/src/main/java/org/torproject/metrics/stats/bwhist/Main.java
@@ -0,0 +1,56 @@
+/* Copyright 2011--2018 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.metrics.stats.bwhist;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.nio.file.Paths;
+import java.util.Arrays;
+
+/**
+ * Coordinate downloading and parsing of descriptors and extraction of
+ * statistically relevant data for later processing with R.
+ */
+public class Main {
+
+ private static Logger log = LoggerFactory.getLogger(Main.class);
+
+ private static String[][] paths = {
+ {"recent", "relay-descriptors", "consensuses"},
+ {"recent", "relay-descriptors", "extra-infos"},
+ {"archive", "relay-descriptors", "consensuses"},
+ {"archive", "relay-descriptors", "extra-infos"}};
+
+ /** Executes this data-processing module. */
+ public static void main(String[] args) throws Exception {
+
+log.info("Starting bwhist module.");
+
+log.info("Reading descriptors and inserting relevant parts into the "
++ "database.");
+File[] descriptorDirectories = Arrays.stream(paths).map((String[] path)
+-> Paths.get(Configuration.descriptors, path).toFile())
+.toArray(File[]::new);
+File historyFile = new File(Configuration.history);
+RelayDescriptorDatabaseImporter database
+= new RelayDescriptorDatabaseImporter(descriptorDirectories,
+historyFile, Configuration.database);
+database.importRelayDescriptors();
+
+log.info("Aggregating database entries.");
+database.aggregate();
+
+log.info("Querying aggregated statistics from the database.");
+new Writer().write(Paths.get(Configuration.output, "bandwidth.csv"),
+database.queryBandwidth());
+
+log.info("Closing database connection.");
+database.closeConnection();
+
+lo
[tor-commits] [metrics-web/release] Replace emptyNull() by more meaningful method.
commit cdab81bdd17f8de3f22c28dd6392e697f823b846
Author: Karsten Loesing
Date: Wed Oct 31 10:12:01 2018 +0100
Replace emptyNull() by more meaningful method.
---
.../torproject/metrics/stats/onionperf/Main.java | 23 +++---
1 file changed, 12 insertions(+), 11 deletions(-)
diff --git a/src/main/java/org/torproject/metrics/stats/onionperf/Main.java
b/src/main/java/org/torproject/metrics/stats/onionperf/Main.java
index c54ee6f..02b70af 100644
--- a/src/main/java/org/torproject/metrics/stats/onionperf/Main.java
+++ b/src/main/java/org/torproject/metrics/stats/onionperf/Main.java
@@ -248,8 +248,8 @@ public class Main {
statistics.add(String.format("%s,%d,%s,%s,%.0f,%.0f,%.0f,%d,%d,%d",
dateFormat.format(rs.getDate("date", calendar)),
rs.getInt("filesize"),
-emptyNull(rs.getString("source")),
-emptyNull(rs.getString("server")),
+getStringFromResultSet(rs, "source"),
+getStringFromResultSet(rs, "server"),
getDoubleFromResultSet(rs, "q1"),
getDoubleFromResultSet(rs, "md"),
getDoubleFromResultSet(rs, "q3"),
@@ -276,7 +276,7 @@ public class Main {
while (rs.next()) {
statistics.add(String.format("%s,%s,%d,%d,%d,%d",
dateFormat.format(rs.getDate("date", calendar)),
-emptyNull(rs.getString("source")),
+getStringFromResultSet(rs, "source"),
rs.getInt("position"),
rs.getInt("q1"),
rs.getInt("md"),
@@ -301,7 +301,7 @@ public class Main {
while (rs.next()) {
statistics.add(String.format("%s,%s,%s,%d,%d,%d",
dateFormat.format(rs.getDate("date", calendar)),
-emptyNull(rs.getString("source")),
+getStringFromResultSet(rs, "source"),
rs.getString("server"),
rs.getInt("q1"),
rs.getInt("md"),
@@ -311,8 +311,13 @@ public class Main {
return statistics;
}
- private static String emptyNull(String text) {
-return null == text ? "" : text;
+ /** Retrieves the String value of the designated column in the
+ * current row of the given ResultSet object, or returns the
+ * empty string if the retrieved value was NULL. */
+ private static String getStringFromResultSet(ResultSet rs, String
columnLabel)
+ throws SQLException {
+String result = rs.getString(columnLabel);
+return null == result ? "" : result;
}
/** Retrieves the double value of the designated column in the
@@ -322,11 +327,7 @@ public class Main {
private static Double getDoubleFromResultSet(ResultSet rs, String
columnLabel)
throws SQLException {
double result = rs.getDouble(columnLabel);
-if (rs.wasNull()) {
- return null;
-} else {
- return result;
-}
+return rs.wasNull() ? null : result;
}
static void writeStatistics(Path webstatsPath, List statistics)
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Only include Running relays in totalcw graph.
commit 9df357886ebfca86da41fb7835282a3b57d249c0
Author: Karsten Loesing
Date: Thu Nov 29 08:45:10 2018 +0100
Only include Running relays in totalcw graph.
Previously we included measured bandwidths of all relays in a vote in
the totalcw graph. Now we only include relays with the Running flag in
the vote.
Implements #28137.
---
src/main/java/org/torproject/metrics/stats/totalcw/Parser.java | 3 ++-
.../metrics/stats/totalcw/TotalcwRelayNetworkStatusVoteTest.java| 6 +++---
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/src/main/java/org/torproject/metrics/stats/totalcw/Parser.java
b/src/main/java/org/torproject/metrics/stats/totalcw/Parser.java
index 893184c..b6a35b4 100644
--- a/src/main/java/org/torproject/metrics/stats/totalcw/Parser.java
+++ b/src/main/java/org/torproject/metrics/stats/totalcw/Parser.java
@@ -19,7 +19,8 @@ class Parser {
RelayNetworkStatusVote vote) {
Long measuredSum = null;
for (NetworkStatusEntry entry : vote.getStatusEntries().values()) {
- if (entry.getMeasured() < 0L) {
+ if (null == entry.getFlags() || !entry.getFlags().contains("Running")
+ || entry.getMeasured() < 0L) {
continue;
}
if (null == measuredSum) {
diff --git
a/src/test/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVoteTest.java
b/src/test/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVoteTest.java
index 11f931d..7c5ecc7 100644
---
a/src/test/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVoteTest.java
+++
b/src/test/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVoteTest.java
@@ -39,15 +39,15 @@ public class TotalcwRelayNetworkStatusVoteTest {
{ "2018-10-15-00-00-00-vote-27102BC123E7AF1D4741AE047E160C91ADC76B21-"
+ "049AB3179B12DACC391F06A10C2A8904E4339D33.part",
ZonedDateTime.parse("2018-10-15T00:00:00Z").toLocalDateTime(),
-"bastet", "27102BC123E7AF1D4741AE047E160C91ADC76B21", 138803L },
+"bastet", "27102BC123E7AF1D4741AE047E160C91ADC76B21", 138700L },
{ "2018-10-15-00-00-00-vote-ED03BB616EB2F60BEC80151114BB25CEF515B226-"
+ "2669AD153408F88E416CE6206D1A75EC3324A2F4.part",
ZonedDateTime.parse("2018-10-15T00:00:00Z").toLocalDateTime(),
-"gabelmoo", "ED03BB616EB2F60BEC80151114BB25CEF515B226", 133441L },
+"gabelmoo", "ED03BB616EB2F60BEC80151114BB25CEF515B226", 133370L },
{ "2018-10-15-00-00-00-vote-EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97-"
+ "38C6A19F78948B689345EE41D7119D76246C4D3E.part",
ZonedDateTime.parse("2018-10-15T00:00:00Z").toLocalDateTime(),
-"Faravahar", "EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97", 158534L }
+"Faravahar", "EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97", 158395L }
});
}
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/release] Remove long unused code from legacy module.
commit ca5fa45df0cfb14801e3556caa93f2cc6d26d790
Author: Karsten Loesing
Date: Tue Nov 13 18:10:18 2018 +0100
Remove long unused code from legacy module.
This includes the lock file, the option to write raw output files for
importing into the database, and a couple boolean config options that
have always been true.
Required changes to existing legacy.config (removals):
ImportDirectoryArchives
KeepDirectoryArchiveImportHistory
WriteRelayDescriptorDatabase
WriteRelayDescriptorsRawFiles
RelayDescriptorRawFilesDirectory
Part of #28116.
---
.../metrics/stats/servers/Configuration.java | 46 +---
.../torproject/metrics/stats/servers/LockFile.java | 61 --
.../org/torproject/metrics/stats/servers/Main.java | 33 +-
.../servers/RelayDescriptorDatabaseImporter.java | 126 +
src/main/resources/legacy.config.template | 20
5 files changed, 10 insertions(+), 276 deletions(-)
diff --git
a/src/main/java/org/torproject/metrics/stats/servers/Configuration.java
b/src/main/java/org/torproject/metrics/stats/servers/Configuration.java
index 76788df..b6ee397 100644
--- a/src/main/java/org/torproject/metrics/stats/servers/Configuration.java
+++ b/src/main/java/org/torproject/metrics/stats/servers/Configuration.java
@@ -24,21 +24,11 @@ public class Configuration {
private static Logger log = LoggerFactory.getLogger(Configuration.class);
- private boolean importDirectoryArchives = false;
-
private List directoryArchivesDirectories = new ArrayList<>();
- private boolean keepDirectoryArchiveImportHistory = false;
-
- private boolean writeRelayDescriptorDatabase = false;
-
private String relayDescriptorDatabaseJdbc =
"jdbc:postgresql://localhost/tordir?user=metrics=password";
- private boolean writeRelayDescriptorsRawFiles = false;
-
- private String relayDescriptorRawFilesDirectory = "pg-import/";
-
/** Initializes this configuration class. */
public Configuration() {
@@ -51,24 +41,10 @@ public class Configuration {
String line = null;
try (BufferedReader br = new BufferedReader(new FileReader(configFile))) {
while ((line = br.readLine()) != null) {
-if (line.startsWith("ImportDirectoryArchives")) {
- this.importDirectoryArchives = Integer.parseInt(
- line.split(" ")[1]) != 0;
-} else if (line.startsWith("DirectoryArchivesDirectory")) {
+if (line.startsWith("DirectoryArchivesDirectory")) {
this.directoryArchivesDirectories.add(new File(line.split(" ")[1]));
-} else if (line.startsWith("KeepDirectoryArchiveImportHistory")) {
- this.keepDirectoryArchiveImportHistory = Integer.parseInt(
- line.split(" ")[1]) != 0;
-} else if (line.startsWith("WriteRelayDescriptorDatabase")) {
- this.writeRelayDescriptorDatabase = Integer.parseInt(
- line.split(" ")[1]) != 0;
} else if (line.startsWith("RelayDescriptorDatabaseJDBC")) {
this.relayDescriptorDatabaseJdbc = line.split(" ")[1];
-} else if (line.startsWith("WriteRelayDescriptorsRawFiles")) {
- this.writeRelayDescriptorsRawFiles = Integer.parseInt(
- line.split(" ")[1]) != 0;
-} else if (line.startsWith("RelayDescriptorRawFilesDirectory")) {
- this.relayDescriptorRawFilesDirectory = line.split(" ")[1];
} else if (!line.startsWith("#") && line.length() > 0) {
log.error("Configuration file contains unrecognized "
+ "configuration key in line '{}'! Exiting!", line);
@@ -93,10 +69,6 @@ public class Configuration {
}
}
- public boolean getImportDirectoryArchives() {
-return this.importDirectoryArchives;
- }
-
/** Returns directories containing archived descriptors. */
public List getDirectoryArchivesDirectories() {
if (this.directoryArchivesDirectories.isEmpty()) {
@@ -108,24 +80,8 @@ public class Configuration {
}
}
- public boolean getKeepDirectoryArchiveImportHistory() {
-return this.keepDirectoryArchiveImportHistory;
- }
-
- public boolean getWriteRelayDescriptorDatabase() {
-return this.writeRelayDescriptorDatabase;
- }
-
public String getRelayDescriptorDatabaseJdbc() {
return this.relayDescriptorDatabaseJdbc;
}
-
- public boolean getWriteRelayDescriptorsRawFiles() {
-return this.writeRelayDescriptorsRawFiles;
- }
-
- public String getRelayDescriptorRawFilesDirectory() {
-return this.relayDescriptorRawFilesDirectory;
- }
}
diff --git a/src/main/java/org/torproject/metrics/stats/servers/LockFile.java
b/src/main/java/org/torproject/metrics/stats/servers/LockFile.java
deleted file mode 100644
inde
[tor-commits] [metrics-web/release] Rename ipv6servers module to servers.
commit c95125a2b329c801db84eea2bdbc4e5118bde9b2
Author: Karsten Loesing
Date: Wed Nov 14 10:48:56 2018 +0100
Rename ipv6servers module to servers.
Part of #28116.
---
build.xml | 12 ++--
.../metrics/stats/ipv6servers/Configuration.java | 18 --
.../metrics/stats/servers/Configuration.java | 18 ++
.../stats/{ipv6servers => servers}/Database.java | 2 +-
.../{ipv6servers => servers}/Ipv6NetworkStatus.java| 2 +-
.../{ipv6servers => servers}/Ipv6ServerDescriptor.java | 2 +-
.../metrics/stats/{ipv6servers => servers}/Main.java | 8
.../metrics/stats/{ipv6servers => servers}/Parser.java | 2 +-
.../metrics/stats/{ipv6servers => servers}/Writer.java | 2 +-
.../Ipv6NetworkStatusTest.java | 6 +++---
.../Ipv6ServerDescriptorTest.java | 14 +++---
.../000a7fe20a17bf5d9839a126b1dff43f998aac6f | 0
.../0018ab4f2f28af683d52f06407edbf7ce1bd3b7d | 0
.../0041dbf9fe846f9765882f7dc8332f94b709e35a | 0
.../01003df74972ce952ebfa390f468ef63c50efa25 | 0
.../018c1229d5f56eebfc1d709d4692673d098800e8 | 0
.../2017-12-04-20-00-00-consensus.part | 0
...90507-1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1.part | 0
.../64dd486d89af14027c9a7b4347a94b74dddb5cdb | 0
.../sql/{ipv6servers => servers}/test-ipv6servers.sql | 0
20 files changed, 43 insertions(+), 43 deletions(-)
diff --git a/build.xml b/build.xml
index a391416..89c8b31 100644
--- a/build.xml
+++ b/build.xml
@@ -105,7 +105,7 @@
depends="init"
description="Run all available database pgTAP tests." >
-
+
@@ -319,7 +319,7 @@
-
+
@@ -416,11 +416,11 @@
-
-
+
+
+ value="org.torproject.metrics.stats.servers.Main" />
@@ -477,7 +477,7 @@
-
+
diff --git
a/src/main/java/org/torproject/metrics/stats/ipv6servers/Configuration.java
b/src/main/java/org/torproject/metrics/stats/ipv6servers/Configuration.java
deleted file mode 100644
index d849cb6..000
--- a/src/main/java/org/torproject/metrics/stats/ipv6servers/Configuration.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/* Copyright 2017--2018 The Tor Project
- * See LICENSE for licensing information */
-
-package org.torproject.metrics.stats.ipv6servers;
-
-/** Configuration options parsed from Java properties with reasonable
hard-coded
- * defaults. */
-class Configuration {
- static String descriptors = System.getProperty("ipv6servers.descriptors",
- "../../shared/in/");
- static String database = System.getProperty("ipv6servers.database",
- "jdbc:postgresql:ipv6servers");
- static String history = System.getProperty("ipv6servers.history",
- "status/read-descriptors");
- static String output = System.getProperty("ipv6servers.output",
- "stats/");
-}
-
diff --git
a/src/main/java/org/torproject/metrics/stats/servers/Configuration.java
b/src/main/java/org/torproject/metrics/stats/servers/Configuration.java
new file mode 100644
index 000..1b7c217
--- /dev/null
+++ b/src/main/java/org/torproject/metrics/stats/servers/Configuration.java
@@ -0,0 +1,18 @@
+/* Copyright 2017--2018 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.metrics.stats.servers;
+
+/** Configuration options parsed from Java properties with reasonable
hard-coded
+ * defaults. */
+class Configuration {
+ static String descriptors = System.getProperty("servers.descriptors",
+ "../../shared/in/");
+ static String database = System.getProperty("servers.database",
+ "jdbc:postgresql:ipv6servers");
+ static String history = System.getProperty("servers.history",
+ "status/read-descriptors");
+ static String output = System.getProperty("servers.output",
+ "stats/");
+}
+
diff --git
a/src/main/java/org/torproject/metrics/stats/ipv6servers/Database.java
b/src/main/java/org/torproject/metrics/stats/servers/Database.java
similarity index 99%
rename from src/main/java/org/torproject/metrics/stats/ipv6servers/Database.java
rename to src/main/java/org/torproject/metrics/stats/servers/Database.java
index b5efe3e..9c9bda3 100644
--- a/src/main/java/org/torproject/metrics/stats/ipv6servers/Database.java
+++ b/src/main/java/org/torproject/metrics/stats/servers/Database.java
@@ -1,7 +1,7 @@
/* Copyright 2017--2018 The Tor Project
* See LICENSE for licensing information */
-package org.torproject.metrics.stats.ipv6servers;
+package org.torproject.metrics.stats.servers;
import java
