[tor-commits] [onionoo/release] Update to latest metrics-base.
commit df7d3369e7418ff381e8f6b9850311fa0187e757 Author: Karsten Loesing Date: Mon Oct 14 16:32:25 2019 +0200 Update to latest metrics-base. --- src/build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build b/src/build index 62a964a..d82fff9 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 62a964a7702086ca6ba0e2307809d4f983163b47 +Subproject commit d82fff984634fe006ac7b0b102e7f48a52ca20d9 ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionoo/release] Increase timeout for testCountryDeDe even more.
commit 1579c1e2c103880129271daf30f49cb2e52f7b10
Author: Karsten Loesing
Date: Fri Oct 4 11:01:52 2019 +0200
Increase timeout for testCountryDeDe even more.
The timeout was again being reached in the GitLab CI environment.
---
.../java/org/torproject/metrics/onionoo/server/ResourceServletTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/src/test/java/org/torproject/metrics/onionoo/server/ResourceServletTest.java
b/src/test/java/org/torproject/metrics/onionoo/server/ResourceServletTest.java
index 935c06e..7df56ea 100644
---
a/src/test/java/org/torproject/metrics/onionoo/server/ResourceServletTest.java
+++
b/src/test/java/org/torproject/metrics/onionoo/server/ResourceServletTest.java
@@ -1134,7 +1134,7 @@ public class ResourceServletTest {
null);
}
- @Test(timeout = 400)
+ @Test(timeout = 800)
public void testCountryDeDe() {
this.assertSummaryDocument(
"/summary?country=de=de", 1, new String[] { "TorkaZ" }, 0,
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionoo/master] Prepare for 7.0-1.21.0 release.
commit 08f4d3a7f628c0adaba11363d997eaf453982c76
Author: Karsten Loesing
Date: Fri Oct 18 08:58:11 2019 +0200
Prepare for 7.0-1.21.0 release.
---
CHANGELOG.md | 3 ++-
build.xml| 2 +-
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3ab6329..d0f9e8d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,7 @@
-# Changes in version 7.0-1.21.0 - 2019-1?-??
+# Changes in version 7.0-1.21.0 - 2019-10-18
* Medium changes
+ - Add previously missing Jetty servlets dependency.
- Fix a bug where old reverse DNS lookups were sometimes not
cleared properly.
diff --git a/build.xml b/build.xml
index a6969f5..91e6130 100644
--- a/build.xml
+++ b/build.xml
@@ -11,7 +11,7 @@
+value="${onionoo.protocol.version}-1.21.0"/>
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionoo/master] Always clear old reverse DNS lookups.
commit 6faa1ab757f16263f7925dbe4c49d6dfb698f265
Author: Karsten Loesing
Date: Thu Oct 3 16:43:21 2019 +0200
Always clear old reverse DNS lookups.
The fix is to distinguish between lookup results being null for a
given fingerprint (because we didn't have enough time to look up an
address) and the empty set (because we received 0 verified or 0
unverified host names in response).
Fixes #31941.
---
CHANGELOG.md | 9 -
.../metrics/onionoo/updater/NodeDetailsStatusUpdater.java | 4 ++--
.../metrics/onionoo/updater/RdnsLookupRequest.java | 14 +++---
.../metrics/onionoo/updater/RdnsLookupWorker.java | 4 ++--
4 files changed, 19 insertions(+), 12 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 696b33d..3ab6329 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,11 @@
-# Changes in version 7.0-1.20.0 - 2019-0?-??
+# Changes in version 7.0-1.21.0 - 2019-1?-??
+
+ * Medium changes
+ - Fix a bug where old reverse DNS lookups were sometimes not
+ cleared properly.
+
+
+# Changes in version 7.0-1.20.0 - 2019-10-04
* Medium changes
- Use Ivy for resolving external dependencies rather than relying
diff --git
a/src/main/java/org/torproject/metrics/onionoo/updater/NodeDetailsStatusUpdater.java
b/src/main/java/org/torproject/metrics/onionoo/updater/NodeDetailsStatusUpdater.java
index 2bdf03f..a5f16e4 100644
---
a/src/main/java/org/torproject/metrics/onionoo/updater/NodeDetailsStatusUpdater.java
+++
b/src/main/java/org/torproject/metrics/onionoo/updater/NodeDetailsStatusUpdater.java
@@ -771,10 +771,10 @@ public class NodeDetailsStatusUpdater implements
DescriptorListener,
verifiedLookupResults.get(nodeStatus.getAddress());
SortedSet unverifiedHostNames =
unverifiedLookupResults.get(nodeStatus.getAddress());
- if (null != verifiedHostNames && !verifiedHostNames.isEmpty()) {
+ if (null != verifiedHostNames) {
this.rdnsVerifiedLookupResults.put(fingerprint, verifiedHostNames);
}
- if (null != unverifiedHostNames && !unverifiedHostNames.isEmpty()) {
+ if (null != unverifiedHostNames) {
this.rdnsUnverifiedLookupResults.put(fingerprint, unverifiedHostNames);
}
}
diff --git
a/src/main/java/org/torproject/metrics/onionoo/updater/RdnsLookupRequest.java
b/src/main/java/org/torproject/metrics/onionoo/updater/RdnsLookupRequest.java
index 2f598a4..53e3fa9 100644
---
a/src/main/java/org/torproject/metrics/onionoo/updater/RdnsLookupRequest.java
+++
b/src/main/java/org/torproject/metrics/onionoo/updater/RdnsLookupRequest.java
@@ -39,9 +39,9 @@ class RdnsLookupRequest extends Thread {
@Override
public void run() {
this.lookupStartedMillis = System.currentTimeMillis();
+final SortedSet verifiedResults = new TreeSet<>();
+final SortedSet unverifiedResults = new TreeSet<>();
try {
- final SortedSet verifiedResults = new TreeSet<>();
- final SortedSet unverifiedResults = new TreeSet<>();
final String[] bytes = this.address.split("\\.");
if (bytes.length == 4) {
final String reverseDnsDomain =
@@ -69,15 +69,15 @@ class RdnsLookupRequest extends Thread {
}
}
}
- synchronized (this) {
-this.verifiedHostNames = verifiedResults;
-this.unverifiedHostNames = unverifiedResults;
- }
} catch (NamingException e) {
/* The Onionoo field is omitted for both lookup failure and absence of
* a host name. We'll try again the next time. */
}
-this.lookupCompletedMillis = System.currentTimeMillis();
+synchronized (this) {
+ this.verifiedHostNames = verifiedResults;
+ this.unverifiedHostNames = unverifiedResults;
+ this.lookupCompletedMillis = System.currentTimeMillis();
+}
this.parent.interrupt();
}
diff --git
a/src/main/java/org/torproject/metrics/onionoo/updater/RdnsLookupWorker.java
b/src/main/java/org/torproject/metrics/onionoo/updater/RdnsLookupWorker.java
index ab5360e..6353daa 100644
--- a/src/main/java/org/torproject/metrics/onionoo/updater/RdnsLookupWorker.java
+++ b/src/main/java/org/torproject/metrics/onionoo/updater/RdnsLookupWorker.java
@@ -40,7 +40,7 @@ class RdnsLookupWorker extends Thread {
/* Getting interrupted should be the default case. */
}
SortedSet verifiedHostNames = request.getVerifiedHostNames();
- if (null != verifiedHostNames && !verifiedHostNames.isEmpty()) {
+ if (null != verifiedHostNames) {
synchronized (this.reverseDomainNameResolver
.rdnsVerifiedLookupResults) {
this.reverseDomainNameResolver.rdnsVerifiedLookupResults.put(
@@ -48,7 +48,7 @@ class RdnsLookupWorker extends Thread {
}
}
SortedSet unverifiedHostNames = request.getUnverifiedHostNames();
- if (n
[tor-commits] [exonerator/master] Print out a warning if we're missing data.
commit b0165cb03138e933a337ddc8400c3a8bbe889b3d
Author: Karsten Loesing
Date: Mon Sep 30 17:44:38 2019 +0200
Print out a warning if we're missing data.
We're now counting the hours for which we have statuses and exit
lists. More precisely, we're truncating consensus valid-after times
and exit scan times to the hour and counting how many distinct hours
we saw during the requested time. If we're missing 18 or more hours of
statuses or exit lists we're printing out a note in the summary.
Fixes #31071.
---
.../metrics/exonerator/ExoneraTorServlet.java | 43 --
.../metrics/exonerator/QueryResponse.java | 21 ++-
.../metrics/exonerator/QueryServlet.java | 24 +++-
src/main/resources/ExoneraTor.properties | 1 +
src/main/sql/exonerator2.sql | 42 +++--
.../metrics/exonerator/ExoneraTorServletTest.java | 2 +-
.../metrics/exonerator/QueryResponseTest.java | 12 --
7 files changed, 112 insertions(+), 33 deletions(-)
diff --git
a/src/main/java/org/torproject/metrics/exonerator/ExoneraTorServlet.java
b/src/main/java/org/torproject/metrics/exonerator/ExoneraTorServlet.java
index a74aa60..4b0cf16 100644
--- a/src/main/java/org/torproject/metrics/exonerator/ExoneraTorServlet.java
+++ b/src/main/java/org/torproject/metrics/exonerator/ExoneraTorServlet.java
@@ -89,6 +89,7 @@ public class ExoneraTorServlet extends HttpServlet {
ExoneraTorDate firstDate = ExoneraTorDate.INVALID;
ExoneraTorDate lastDate = ExoneraTorDate.INVALID;
boolean noRelevantConsensuses = true;
+ boolean missingData = false;
List statusEntries = new ArrayList<>();
List addressesInSameNetwork = null;
@@ -105,6 +106,14 @@ public class ExoneraTorServlet extends HttpServlet {
&& queryResponse.relevantStatuses) {
noRelevantConsensuses = false;
}
+ if (null != queryResponse.missingStatuses
+ && queryResponse.missingStatuses) {
+missingData = true;
+ }
+ if (null != queryResponse.missingExitLists
+ && queryResponse.missingExitLists) {
+missingData = true;
+ }
if (null != queryResponse.matches) {
for (QueryResponse.Match match : queryResponse.matches) {
StringBuilder sb = new StringBuilder();
@@ -210,15 +219,18 @@ public class ExoneraTorServlet extends HttpServlet {
/* Print out result. */
} else {
if (!statusEntries.isEmpty()) {
- this.writeSummaryPositive(out, rb, relayIp, requestedDate.asString);
+ this.writeSummaryPositive(out, rb, relayIp, requestedDate.asString,
+ missingData);
this.writeTechnicalDetails(out, rb, relayIp, requestedDate.asString,
statusEntries);
} else if (addressesInSameNetwork != null
&& !addressesInSameNetwork.isEmpty()) {
this.writeSummaryAddressesInSameNetwork(out, rb, requestUrl, relayIp,
- requestedDate.asString, langStr, addressesInSameNetwork);
+ requestedDate.asString, langStr, addressesInSameNetwork,
+ missingData);
} else {
- this.writeSummaryNegative(out, rb, relayIp, requestedDate.asString);
+ this.writeSummaryNegative(out, rb, relayIp, requestedDate.asString,
+ missingData);
}
this.writePermanentLink(out, rb, requestUrl, relayIp,
requestedDate.asString, langStr);
@@ -398,13 +410,13 @@ public class ExoneraTorServlet extends HttpServlet {
private void writeSummaryNoTimestamp(PrintWriter out, ResourceBundle rb) {
this.writeSummary(out, rb.getString("summary.heading"),
"panel-danger",
-rb.getString("summary.invalidparams.notimestamp.title"), null,
+rb.getString("summary.invalidparams.notimestamp.title"), null, null,
rb.getString("summary.invalidparams.notimestamp.body"));
}
private void writeSummaryNoIp(PrintWriter out, ResourceBundle rb) {
this.writeSummary(out, rb.getString("summary.heading"),
-"panel-danger", rb.getString("summary.invalidparams.noip.title"),
+"panel-danger", rb.getString("summary.invalidparams.noip.title"), null,
null, rb.getString("summary.invalidparams.noip.body"));
}
@@ -413,7 +425,7 @@ public class ExoneraTorServlet extends HttpServlet {
String lastDate) {
this.writeSummary(out, rb.getString("summary.heading"),
"panel-danger",
-rb.getString("summary.invalidparams.timestamprange.title"), null,
+rb.getString("summary.invalidparams.timestamprange.title"), null, null,
rb.getString("summary.invalid
[tor-commits] [exonerator/master] Make Jetty listen host configurable.
commit c6f7b83915888c9b4b61a8aa6ae39ec1b2248b71 Author: Karsten Loesing Date: Mon Sep 30 17:43:22 2019 +0200 Make Jetty listen host configurable. The default listen host is still 127.0.0.1, but now it's possible to configure a different listen host, like 0.0.0.0, which can facilitate local testing. --- src/main/resources/jetty.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/resources/jetty.xml b/src/main/resources/jetty.xml index e1e4d4d..56887f7 100644 --- a/src/main/resources/jetty.xml +++ b/src/main/resources/jetty.xml @@ -75,7 +75,9 @@ -127.0.0.1 + + + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [exonerator/master] Update to latest metrics-base.
commit 45c5285a3514d7c5acc3fcf040eb18dcfd8b7e16 Author: Karsten Loesing Date: Mon Sep 30 17:42:33 2019 +0200 Update to latest metrics-base. --- src/build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build b/src/build index 07c2a00..62a964a 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 07c2a00c27f0d536223f8b5a61fc91e60eb524d4 +Subproject commit 62a964a7702086ca6ba0e2307809d4f983163b47 ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionoo/master] Update to latest metrics-base.
commit df7d3369e7418ff381e8f6b9850311fa0187e757 Author: Karsten Loesing Date: Mon Oct 14 16:32:25 2019 +0200 Update to latest metrics-base. --- src/build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build b/src/build index 62a964a..d82fff9 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 62a964a7702086ca6ba0e2307809d4f983163b47 +Subproject commit d82fff984634fe006ac7b0b102e7f48a52ca20d9 ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-base/master] Add missing Jetty servlets dependency.
commit d82fff984634fe006ac7b0b102e7f48a52ca20d9 Author: Karsten Loesing Date: Mon Oct 14 16:30:13 2019 +0200 Add missing Jetty servlets dependency. Fixes #32062. --- java/ivy.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/java/ivy.xml b/java/ivy.xml index 19b2f4b..851c5d3 100644 --- a/java/ivy.xml +++ b/java/ivy.xml @@ -40,6 +40,7 @@ + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionoo/master] Bump version to 7.0-1.20.0-dev.
commit c61ce6c4ec69741c3fe35f48e4cddf18f5a8ee49
Author: Karsten Loesing
Date: Fri Oct 4 11:09:15 2019 +0200
Bump version to 7.0-1.20.0-dev.
---
build.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/build.xml b/build.xml
index 360dc8b..a6969f5 100644
--- a/build.xml
+++ b/build.xml
@@ -11,7 +11,7 @@
+value="${onionoo.protocol.version}-1.20.0-dev"/>
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionoo/release] Adds GitLab CI for tests and checks
commit 5173fe1d73c32342be937160d4974bd446c45e57 Author: Iain R. Learmonth Date: Thu Sep 5 12:26:45 2019 +0100 Adds GitLab CI for tests and checks Configuration is held in the .gitlab-ci.yml file. Fixes: #31400 --- .gitlab-ci.yml | 25 + 1 file changed, 25 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000..b67b5a4 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,25 @@ +variables: + GIT_STRATEGY: clone + METRICS_LIB_VERSION: "2.6.2" + JAVA_TOOL_OPTIONS: "-Dfile.encoding=UTF-8" + +stages: + - test + +test: + stage: test + image: debian:buster + script: + - apt update + - apt -y install default-jdk ant ivy git curl + - git submodule init + - git submodule update + - mkdir lib + - mkdir tmp + - pushd tmp + - curl https://dist.torproject.org/metrics-lib/$METRICS_LIB_VERSION/metrics-lib-$METRICS_LIB_VERSION.tar.gz | tar xzf - + - popd + - mv tmp/metrics-lib-$METRICS_LIB_VERSION/generated/dist/metrics-lib-$METRICS_LIB_VERSION-thin.jar lib + - ant -lib /usr/share/java resolve + - ant test + - ant checks ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionoo/release] Make unit test locale-independent.
commit 860228c58f294429a44793aa91b30a93010ad8f1
Author: Karsten Loesing
Date: Tue Aug 6 17:17:05 2019 +0200
Make unit test locale-independent.
Spotted while working on #31326, but unrelated.
---
src/main/java/org/torproject/metrics/onionoo/docs/WeightsStatus.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git
a/src/main/java/org/torproject/metrics/onionoo/docs/WeightsStatus.java
b/src/main/java/org/torproject/metrics/onionoo/docs/WeightsStatus.java
index d0f50b6..121ec87 100644
--- a/src/main/java/org/torproject/metrics/onionoo/docs/WeightsStatus.java
+++ b/src/main/java/org/torproject/metrics/onionoo/docs/WeightsStatus.java
@@ -7,6 +7,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Comparator;
+import java.util.Locale;
import java.util.Map;
import java.util.Scanner;
import java.util.SortedMap;
@@ -206,7 +207,7 @@ public class WeightsStatus extends Document {
for (int i = 0; i < weights.length; i++) {
sb.append(" ");
if (i != 0 && i != 5 && !Double.valueOf(weights[i]).isNaN()) {
- sb.append(String.format("%.12f", weights[i]));
+ sb.append(String.format(Locale.US, "%.12f", weights[i]));
}
}
sb.append("\n");
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionoo/release] Remove Cobertura from the build process.
commit 301841c8e97602fe8106cb1bbd1c46a19a767a63 Author: Karsten Loesing Date: Wed Aug 14 11:42:59 2019 +0200 Remove Cobertura from the build process. --- CHANGELOG.md | 1 + src/build| 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c8062b..696b33d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ copied to the `lib/` directory manually. Current dependency versions resolved by Ivy are the same as in Debian stretch with few exceptions. + - Remove Cobertura from the build process. # Changes in version 7.0-1.19.1 - 2018-11-20 diff --git a/src/build b/src/build index 256e619..75cda35 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 256e6192ad0066e48c6abb0ee9ac71714c7a3a0f +Subproject commit 75cda35ba7375ab0f5b12fa214620c3a29917097 ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionoo/release] Bump version to 7.0-1.19.1-dev.
commit 6ff4af1d6ce370f1f676715cef084704927c0ceb
Author: Karsten Loesing
Date: Tue Nov 20 17:05:20 2018 +0100
Bump version to 7.0-1.19.1-dev.
---
build.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/build.xml b/build.xml
index d066bcb..1f6b69c 100644
--- a/build.xml
+++ b/build.xml
@@ -10,7 +10,7 @@
+value="${onionoo.protocol.version}-1.19.1-dev"/>
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionoo/release] Prepare for 7.0-1.20.0 release.
commit aa6889d44bd06c4e5b9b5b86c3d2f3dd0fae5b5b
Author: Karsten Loesing
Date: Fri Oct 4 10:46:44 2019 +0200
Prepare for 7.0-1.20.0 release.
---
build.xml | 2 +-
src/build | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/build.xml b/build.xml
index b0f30f8..360dc8b 100644
--- a/build.xml
+++ b/build.xml
@@ -11,7 +11,7 @@
+value="${onionoo.protocol.version}-1.20.0"/>
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionoo/release] Increase timeout for testCountryDeDe
commit 4ba7b00c6f4080955af907e1dbf9dd0c79fb9a58
Author: Iain R. Learmonth
Date: Thu Sep 5 14:10:44 2019 +0100
Increase timeout for testCountryDeDe
The timeout was being reached in the GitLab CI environment, which is not
super powerful.
---
.../java/org/torproject/metrics/onionoo/server/ResourceServletTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/src/test/java/org/torproject/metrics/onionoo/server/ResourceServletTest.java
b/src/test/java/org/torproject/metrics/onionoo/server/ResourceServletTest.java
index 7c3c52e..935c06e 100644
---
a/src/test/java/org/torproject/metrics/onionoo/server/ResourceServletTest.java
+++
b/src/test/java/org/torproject/metrics/onionoo/server/ResourceServletTest.java
@@ -1134,7 +1134,7 @@ public class ResourceServletTest {
null);
}
- @Test(timeout = 200)
+ @Test(timeout = 400)
public void testCountryDeDe() {
this.assertSummaryDocument(
"/summary?country=de=de", 1, new String[] { "TorkaZ" }, 0,
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionoo/release] Use metrics-base ant task to fetch metrics-lib.
commit bbc5c16d7b748ab05707a31779849f216a89f1b5 Author: Karsten Loesing Date: Fri Sep 27 10:06:13 2019 +0200 Use metrics-base ant task to fetch metrics-lib. Implements part of #31649. --- .gitlab-ci.yml | 7 +-- src/build | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b67b5a4..897c9da 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,5 @@ variables: GIT_STRATEGY: clone - METRICS_LIB_VERSION: "2.6.2" JAVA_TOOL_OPTIONS: "-Dfile.encoding=UTF-8" stages: @@ -15,11 +14,7 @@ test: - git submodule init - git submodule update - mkdir lib - - mkdir tmp - - pushd tmp - - curl https://dist.torproject.org/metrics-lib/$METRICS_LIB_VERSION/metrics-lib-$METRICS_LIB_VERSION.tar.gz | tar xzf - - - popd - - mv tmp/metrics-lib-$METRICS_LIB_VERSION/generated/dist/metrics-lib-$METRICS_LIB_VERSION-thin.jar lib + - ant fetch-metrics-lib - ant -lib /usr/share/java resolve - ant test - ant checks diff --git a/src/build b/src/build index 75cda35..07c2a00 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 75cda35ba7375ab0f5b12fa214620c3a29917097 +Subproject commit 07c2a00c27f0d536223f8b5a61fc91e60eb524d4 ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionoo/release] Stop signing jars.
commit 43ad7e350a4a2c0ec822636a78db8d669f871686 Author: Karsten Loesing Date: Wed Nov 28 10:19:00 2018 +0100 Stop signing jars. Implements #28584. --- CERT | 21 - src/build | 2 +- 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/CERT b/CERT deleted file mode 100644 index b90b397..000 --- a/CERT +++ /dev/null @@ -1,21 +0,0 @@ --BEGIN CERTIFICATE- -MIIDaTCCAlGgAwIBAgIELle0dTANBgkqhkiG9w0BAQsFADBlMQswCQYDVQQGEwJV -UzELMAkGA1UECBMCV0ExEDAOBgNVBAcTB1NlYXR0bGUxHTAbBgNVBAoTFFRoZSBU -b3IgUHJvamVjdCwgSW5jMRgwFgYDVQQDEw9LYXJzdGVuIExvZXNpbmcwHhcNMTgw -ODI4MDcwNjM2WhcNMTgxMTI2MDcwNjM2WjBlMQswCQYDVQQGEwJVUzELMAkGA1UE -CBMCV0ExEDAOBgNVBAcTB1NlYXR0bGUxHTAbBgNVBAoTFFRoZSBUb3IgUHJvamVj -dCwgSW5jMRgwFgYDVQQDEw9LYXJzdGVuIExvZXNpbmcwggEiMA0GCSqGSIb3DQEB -AQUAA4IBDwAwggEKAoIBAQChXn+IUp+o6G+k4ffxk3TkxZb3iXfiG7byNsG63olU -6aTpAjDMeaT4ctUwxH4+56Sbcf/wB0vEFBbX8MyRd1eY02PKwMVJ6VBhjOQcIlrd -Qw+VAhKTcEIv4yiR0BWapQyR07pgmKirYVjN6s6ef8NJzUptpxLlaYJ3ZfQfc4aE -MXzScgaccwDFIWQ661lzLGCfeSxxa3Xy4wWsGwzNzLITYrrABcbg7yogLo2btNvD -oEwGL3/baQdhl0dra6biVCZr9ydn3Hg57S55pUU0rBY25id78zUO8xrfNHw54wwX -lOblGt75OOkahP/ZZSBxxoiknJ6y5VQV8y+noA4vigXFAgMBAAGjITAfMB0GA1Ud -DgQWBBSeh60M+/wMYyYhlxtuff2Hk9n7bzANBgkqhkiG9w0BAQsFAAOCAQEAkXZs -3T3GTkZ+EGvZG5puzKdgZiSsLgIy25xdWsIx147AIZEJFKjEAtbu0osMpkTa96B6 -a+BHf7PTjQUuH3YOEmeW9ab8pwu5SRijCq2qkuvjjSLBcJzWnalcKDYYvoQte1// -Di8JqpRXCw20WY2bldTiafyG80E0RGfiX2I8vbDiPIhjwz9Wox8Q1rw1c9T/vRn9 -pI8FrHgTnDO6R54yD25QSpsj+hC+IDkFKO17vGCIaJrPG5o6th438ijEwJsG+LRB -4zKKKsFTby7UJI3Ag8xolIhsBkRZO2j4Na35i15SZ7QJNj9J5g171z8RyOmyIQbg -q7OXN2iiRIxiIJwoQw== --END CERTIFICATE- diff --git a/src/build b/src/build index 08514a3..e639c69 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 08514a32afefbeef848b80f9a338ee840c282604 +Subproject commit e639c697e9e94c6dbb26e946e5247c20a62c0661 ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionoo/release] Use Ivy for resolving external dependencies.
commit 8c3b87b92137f85391baf21ecef749561a97a9e3 Author: Karsten Loesing Date: Tue Aug 6 17:21:20 2019 +0200 Use Ivy for resolving external dependencies. Includes a metrics-lib upgrade to 2.6.2 and an upgrade to the latest metrics-base commit 256e619. Implements part of #31326. --- CHANGELOG.md | 14 ++ build.xml| 29 +++-- src/build| 2 +- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d3fb72..9c8062b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ +# Changes in version 7.0-1.20.0 - 2019-0?-?? + + * Medium changes + - Use Ivy for resolving external dependencies rather than relying + on files found in Debian stable packages. Requires installing Ivy + (using `apt-get install ivy`, `brew install ivy`, or similar) and + running `ant resolve` (or `ant -lib /usr/share/java resolve`). + Retrieved files are then copied to the `lib/` directory, except + for dependencies on other metrics libraries that still need to be + copied to the `lib/` directory manually. Current dependency + versions resolved by Ivy are the same as in Debian stretch with + few exceptions. + + # Changes in version 7.0-1.19.1 - 2018-11-20 * Minor changes diff --git a/build.xml b/build.xml index 1f6b69c..b0f30f8 100644 --- a/build.xml +++ b/build.xml @@ -4,14 +4,15 @@ ]> - + - + @@ -43,7 +44,7 @@ - + @@ -57,17 +58,17 @@ - - - - - - - - - - - + + + + + + + + + + + diff --git a/src/build b/src/build index e639c69..256e619 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit e639c697e9e94c6dbb26e946e5247c20a62c0661 +Subproject commit 256e6192ad0066e48c6abb0ee9ac71714c7a3a0f ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionoo/master] Increase timeout for testCountryDeDe even more.
commit 1579c1e2c103880129271daf30f49cb2e52f7b10
Author: Karsten Loesing
Date: Fri Oct 4 11:01:52 2019 +0200
Increase timeout for testCountryDeDe even more.
The timeout was again being reached in the GitLab CI environment.
---
.../java/org/torproject/metrics/onionoo/server/ResourceServletTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/src/test/java/org/torproject/metrics/onionoo/server/ResourceServletTest.java
b/src/test/java/org/torproject/metrics/onionoo/server/ResourceServletTest.java
index 935c06e..7df56ea 100644
---
a/src/test/java/org/torproject/metrics/onionoo/server/ResourceServletTest.java
+++
b/src/test/java/org/torproject/metrics/onionoo/server/ResourceServletTest.java
@@ -1134,7 +1134,7 @@ public class ResourceServletTest {
null);
}
- @Test(timeout = 400)
+ @Test(timeout = 800)
public void testCountryDeDe() {
this.assertSummaryDocument(
"/summary?country=de=de", 1, new String[] { "TorkaZ" }, 0,
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionoo/master] Prepare for 7.0-1.20.0 release.
commit aa6889d44bd06c4e5b9b5b86c3d2f3dd0fae5b5b
Author: Karsten Loesing
Date: Fri Oct 4 10:46:44 2019 +0200
Prepare for 7.0-1.20.0 release.
---
build.xml | 2 +-
src/build | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/build.xml b/build.xml
index b0f30f8..360dc8b 100644
--- a/build.xml
+++ b/build.xml
@@ -11,7 +11,7 @@
+value="${onionoo.protocol.version}-1.20.0"/>
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/master] Add "Tor Browser updates by release channel" graph.
commit cba1711544fbfea9e88ba4161f94b26231b28d5f
Author: Karsten Loesing
Date: Wed Oct 2 12:24:38 2019 +0200
Add "Tor Browser updates by release channel" graph.
Implements #31755.
---
src/main/R/rserver/rserve-init.R | 46 ++
src/main/resources/web.xml | 4 ++
src/main/resources/web/json/categories.json| 1 +
src/main/resources/web/json/metrics.json | 11 ++
.../resources/web/jsps/reproducible-metrics.jsp| 1 +
src/main/resources/web/jsps/stats.jsp | 22 +++
6 files changed, 85 insertions(+)
diff --git a/src/main/R/rserver/rserve-init.R b/src/main/R/rserver/rserve-init.R
index af74ef7..5eac4d7 100644
--- a/src/main/R/rserver/rserve-init.R
+++ b/src/main/R/rserver/rserve-init.R
@@ -1399,6 +1399,52 @@ plot_webstats_tb_locale <- function(start_p, end_p,
path_p) {
ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}
+prepare_webstats_tb_channel <- function(start_p = NULL, end_p = NULL) {
+ read_csv(file = paste(stats_dir, "webstats.csv", sep = ""),
+ col_types = cols(
+log_date = col_date(format = ""),
+request_type = col_factor(levels = NULL),
+platform = col_skip(),
+channel = col_factor(levels = NULL),
+locale = col_skip(),
+incremental = col_skip(),
+count = col_double())) %>%
+filter(if (!is.null(start_p)) log_date >= as.Date(start_p) else TRUE) %>%
+filter(if (!is.null(end_p)) log_date <= as.Date(end_p) else TRUE) %>%
+filter(request_type %in% c("tbup", "tbur")) %>%
+filter(channel %in% c("a", "r")) %>%
+group_by(log_date, channel, request_type) %>%
+summarize(count = sum(count)) %>%
+dcast(log_date + channel ~ request_type, value.var = "count") %>%
+rename(date = log_date, update_pings = tbup, update_requests = tbur)
+}
+
+plot_webstats_tb_channel <- function(start_p, end_p, path_p) {
+ prepare_webstats_tb_channel(start_p, end_p) %>%
+gather(request_type, count, -c(date, channel)) %>%
+unite("request_type_channel", request_type, channel) %>%
+mutate(request_type_channel = factor(request_type_channel,
+ levels = c("update_pings_r", "update_pings_a",
+ "update_requests_r", "update_requests_a"),
+ labels = c("Update pings (stable)", "Update pings (alpha)",
+ "Update requests (stable)", "Update requests (alpha)"))) %>%
+ungroup() %>%
+complete(date = full_seq(date, period = 1),
+ nesting(request_type_channel)) %>%
+ggplot(aes(x = date, y = count)) +
+geom_point() +
+geom_line() +
+scale_x_date(name = "", breaks = custom_breaks,
+ labels = custom_labels, minor_breaks = custom_minor_breaks) +
+scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) +
+facet_grid(request_type_channel ~ ., scales = "free_y") +
+theme(strip.text.y = element_text(angle = 0, hjust = 0, size = rel(1.5)),
+ strip.background = element_rect(fill = NA)) +
+ggtitle("Tor Browser updates by release channel") +
+labs(caption = copyright_notice)
+ ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
+}
+
prepare_webstats_tm <- function(start_p = NULL, end_p = NULL) {
read_csv(file = paste(stats_dir, "webstats.csv", sep = ""),
col_types = cols(
diff --git a/src/main/resources/web.xml b/src/main/resources/web.xml
index 2c68fd7..10d12ee 100644
--- a/src/main/resources/web.xml
+++ b/src/main/resources/web.xml
@@ -51,6 +51,7 @@
/webstats-tb.html
/webstats-tb-platform.html
/webstats-tb-locale.html
+/webstats-tb-channel.html
/webstats-tm.html
/relays-ipv6.html
/bridges-ipv6.html
@@ -182,6 +183,9 @@
/webstats-tb-locale.png
/webstats-tb-locale.pdf
/webstats-tb-locale.csv
+/webstats-tb-channel.png
+/webstats-tb-channel.pdf
+/webstats-tb-channel.csv
/webstats-tm.png
/webstats-tm.pdf
/webstats-tm.csv
diff --git a/src/main/resources/web/json/categories.json
b/src/main/resources/web/json/categories.json
index 89742bc..3771631 100644
--- a/src/main/resources/web/json/categories.json
+++ b/src/main/resources/web/json/categories.json
@@ -88,6 +88,7 @@
"webstats-tb",
"webstats-tb-platform",
"webstats-tb-locale",
+ "webstats-tb-channel",
"webstats-tm"
]
}
diff --git a/src/main/resources/web/json/metrics.json
b/src/main/resources/web/json/metrics.json
index e6bab04..54ce78b 100644
--- a/src/main/resources/web/json/metrics.json
+++ b/src/main/resources/web/json/metrics.json
@@ -416,6 +416,17 @@
]
},
[tor-commits] [metrics-web/master] Upgrade to latest metrics-base and -lib.
commit e2940260a1879d3a4354e826108366746b10a243 Author: Karsten Loesing Date: Wed Oct 2 14:35:54 2019 +0200 Upgrade to latest metrics-base and -lib. --- src/build | 2 +- src/submods/metrics-lib | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/build b/src/build index 07c2a00..62a964a 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 07c2a00c27f0d536223f8b5a61fc91e60eb524d4 +Subproject commit 62a964a7702086ca6ba0e2307809d4f983163b47 diff --git a/src/submods/metrics-lib b/src/submods/metrics-lib index c1581dd..8e2f671 16 --- a/src/submods/metrics-lib +++ b/src/submods/metrics-lib @@ -1 +1 @@ -Subproject commit c1581dd8ca26798dc90ea2189ca41636b50e1e0d +Subproject commit 8e2f67107f1e6add0db3fa209cb89c3b69f7bf06 ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-lib/master] Upgrade to latest metrics-base.
commit 8e2f67107f1e6add0db3fa209cb89c3b69f7bf06 Author: Karsten Loesing Date: Wed Oct 2 14:34:41 2019 +0200 Upgrade to latest metrics-base. --- src/build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build b/src/build index 07c2a00..62a964a 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 07c2a00c27f0d536223f8b5a61fc91e60eb524d4 +Subproject commit 62a964a7702086ca6ba0e2307809d4f983163b47 ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/master] Tweak URL-based unit tests.
commit 834ef7c5e76778e31f9ca8569e69423b32700acf
Author: Karsten Loesing
Date: Fri Sep 27 12:23:19 2019 +0200
Tweak URL-based unit tests.
Turns out that one of the tests that we disabled earlier was only
slow, because creating a URL instance to https://something.dummy.org/
triggered an actual DNS resolve. There's absolutely no need to do
this, so we can as well use localhost in all our test URLs.
---
.../org/torproject/metrics/collector/conf/ConfigurationTest.java | 5 ++---
.../torproject/metrics/collector/downloader/DownloaderTest.java | 8
2 files changed, 6 insertions(+), 7 deletions(-)
diff --git
a/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java
b/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java
index 67a9082..7845909 100644
--- a/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java
+++ b/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java
@@ -128,13 +128,12 @@ public class ConfigurationTest {
}
}
- @Ignore("This test takes 40 seconds, which is too long.")
@Test()
public void testUrlArrayValues() throws Exception {
URL[] array = new URL[randomSource.nextInt(30) + 1];
for (int i = 0; i < array.length; i++) {
- array[i] = new URL("https://;
- + Integer.toBinaryString(randomSource.nextInt(100)) + ".dummy.org");
+ array[i] = new URL("http://localhost/;
+ + Integer.toBinaryString(randomSource.nextInt(100)));
}
String input =
Arrays.toString(array).replace("[", "").replace("]", "")
diff --git
a/src/test/java/org/torproject/metrics/collector/downloader/DownloaderTest.java
b/src/test/java/org/torproject/metrics/collector/downloader/DownloaderTest.java
index 19db782..0617940 100644
---
a/src/test/java/org/torproject/metrics/collector/downloader/DownloaderTest.java
+++
b/src/test/java/org/torproject/metrics/collector/downloader/DownloaderTest.java
@@ -85,7 +85,7 @@ public class DownloaderTest {
@Test
public void testExistingResource() throws Exception {
-URL requestedUrl = new URL("http://example.org/exists;);
+URL requestedUrl = new URL("http://localhost/exists;);
byte[] expectedDownloadedBytes = "content".getBytes();
HttpURLConnection urlConnection = mock(HttpURLConnection.class);
httpUrlStreamHandler.addConnection(requestedUrl, urlConnection);
@@ -98,7 +98,7 @@ public class DownloaderTest {
@Test
public void testNonExistingResource() throws Exception {
-URL requestedUrl = new URL("http://example.org/notfound;);
+URL requestedUrl = new URL("http://localhost/notfound;);
HttpURLConnection urlConnection = mock(HttpURLConnection.class);
httpUrlStreamHandler.addConnection(requestedUrl, urlConnection);
given(urlConnection.getResponseCode()).willReturn(404);
@@ -108,7 +108,7 @@ public class DownloaderTest {
@Test
public void testEmptyResource() throws Exception {
-URL requestedUrl = new URL("http://example.org/empty;);
+URL requestedUrl = new URL("http://localhost/empty;);
byte[] expectedDownloadedBytes = new byte[0];
HttpURLConnection urlConnection = mock(HttpURLConnection.class);
httpUrlStreamHandler.addConnection(requestedUrl, urlConnection);
@@ -121,7 +121,7 @@ public class DownloaderTest {
@Test(expected = SocketTimeoutException.class)
public void testTimeout() throws Exception {
-URL requestedUrl = new URL("http://example.org/timeout;);
+URL requestedUrl = new URL("http://localhost/timeout;);
SocketTimeoutException expectedException = new SocketTimeoutException();
HttpURLConnection urlConnection = mock(HttpURLConnection.class);
httpUrlStreamHandler.addConnection(requestedUrl, urlConnection);
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/master] Create utility class for downloading from HTTP servers
commit dc3a2e4aea03ca44be033dafc3dcc3d7e0d5c07f
Author: Karsten Loesing
Date: Thu Sep 26 11:48:18 2019 +0200
Create utility class for downloading from HTTP servers
Also skip two tests in ConfigurationTest that together take 53 seconds
which has the effect that we don't run tests very often.
Implements #31599.
---
CHANGELOG.md | 6 +
src/build | 2 +-
.../metrics/collector/downloader/Downloader.java | 62 ++
.../collector/exitlists/ExitListDownloader.java| 33 ++---
.../collector/onionperf/OnionPerfDownloader.java | 23 +++-
.../relaydescs/RelayDescriptorDownloader.java | 32 +
.../snowflake/SnowflakeStatsDownloader.java| 56 ++---
.../metrics/collector/conf/ConfigurationTest.java | 3 +
.../collector/downloader/DownloaderTest.java | 134 +
9 files changed, 250 insertions(+), 101 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e30b086..6e72f02 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,9 @@
+# Changes in version 1.12.0 - 2019-??-??
+
+ * Medium changes
+ - Require Mockito 1.10.19 as dependency for running tests.
+
+
# Changes in version 1.11.1 - 2019-09-19
* Minor changes
diff --git a/src/build b/src/build
index 07c2a00..62a964a 16
--- a/src/build
+++ b/src/build
@@ -1 +1 @@
-Subproject commit 07c2a00c27f0d536223f8b5a61fc91e60eb524d4
+Subproject commit 62a964a7702086ca6ba0e2307809d4f983163b47
diff --git
a/src/main/java/org/torproject/metrics/collector/downloader/Downloader.java
b/src/main/java/org/torproject/metrics/collector/downloader/Downloader.java
new file mode 100644
index 000..a1f3852
--- /dev/null
+++ b/src/main/java/org/torproject/metrics/collector/downloader/Downloader.java
@@ -0,0 +1,62 @@
+/* Copyright 2019 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.metrics.collector.downloader;
+
+import java.io.BufferedInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.zip.InflaterInputStream;
+
+/**
+ * Utility class for downloading resources from HTTP servers.
+ */
+public class Downloader {
+
+ /**
+ * Download the given URL from an HTTP server and return downloaded bytes.
+ *
+ * @param url URL to download.
+ * @return Downloaded bytes, or {@code null} if the resource was not found.
+ * @throws IOException Thrown if anything goes wrong while downloading.
+ */
+ public static byte[] downloadFromHttpServer(URL url) throws IOException {
+return Downloader.downloadFromHttpServer(url, false);
+ }
+
+ /**
+ * Download the given URL from an HTTP server, possibly inflate the response,
+ * and return downloaded bytes.
+ *
+ * @param url URL to download.
+ * @param isDeflated Whether the response is deflated.
+ * @return Downloaded bytes, or {@code null} if the resource was not found.
+ * @throws IOException Thrown if anything goes wrong while downloading.
+ */
+ public static byte[] downloadFromHttpServer(URL url, boolean isDeflated)
+ throws IOException {
+ByteArrayOutputStream downloadedBytes = new ByteArrayOutputStream();
+HttpURLConnection huc = (HttpURLConnection) url.openConnection();
+huc.setRequestMethod("GET");
+huc.setReadTimeout(5000);
+huc.connect();
+int response = huc.getResponseCode();
+if (response != 200) {
+ return null;
+}
+try (BufferedInputStream in = isDeflated
+? new BufferedInputStream(new InflaterInputStream(
+huc.getInputStream()))
+: new BufferedInputStream(huc.getInputStream())) {
+ int len;
+ byte[] data = new byte[1024];
+ while ((len = in.read(data, 0, 1024)) >= 0) {
+downloadedBytes.write(data, 0, len);
+ }
+}
+return downloadedBytes.toByteArray();
+ }
+}
+
diff --git
a/src/main/java/org/torproject/metrics/collector/exitlists/ExitListDownloader.java
b/src/main/java/org/torproject/metrics/collector/exitlists/ExitListDownloader.java
index 66fc1a7..b4bee15 100644
---
a/src/main/java/org/torproject/metrics/collector/exitlists/ExitListDownloader.java
+++
b/src/main/java/org/torproject/metrics/collector/exitlists/ExitListDownloader.java
@@ -12,16 +12,15 @@ import org.torproject.metrics.collector.conf.Configuration;
import org.torproject.metrics.collector.conf.ConfigurationException;
import org.torproject.metrics.collector.conf.Key;
import org.torproject.metrics.collector.cron.CollecTorMain;
+import org.torproject.metrics.collector.downloader.Downloader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.BufferedInputStream;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
-import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.file.P
[tor-commits] [metrics-base/master] Add Mockito as dependency.
commit 62a964a7702086ca6ba0e2307809d4f983163b47
Author: Karsten Loesing
Date: Fri Sep 27 10:30:02 2019 +0200
Add Mockito as dependency.
Required for #31599, but likely useful for other purposes as well.
---
java/base.xml | 2 ++
java/ivy.xml | 1 +
java/junittest.policy | 3 +++
3 files changed, 6 insertions(+)
diff --git a/java/base.xml b/java/base.xml
index 4ce1660..6cb1693 100644
--- a/java/base.xml
+++ b/java/base.xml
@@ -51,6 +51,8 @@
+
+
diff --git a/java/ivy.xml b/java/ivy.xml
index 2860b74..19b2f4b 100644
--- a/java/ivy.xml
+++ b/java/ivy.xml
@@ -18,6 +18,7 @@
+
diff --git a/java/junittest.policy b/java/junittest.policy
index e2f4a26..eb3ea91 100644
--- a/java/junittest.policy
+++ b/java/junittest.policy
@@ -14,4 +14,7 @@ grant {
permission java.lang.RuntimePermission "accessClassInPackage.sun.reflect";
permission java.lang.RuntimePermission
"accessClassInPackage.sun.net.www.protocol.http";
permission java.lang.RuntimePermission
"accessClassInPackage.sun.net.www.http";
+ permission java.lang.RuntimePermission "getProtectionDomain";
+ permission java.lang.RuntimePermission "reflectionFactoryAccess";
+ permission java.lang.RuntimePermission "setFactory";
};
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionoo/master] Use metrics-base ant task to fetch metrics-lib.
commit bbc5c16d7b748ab05707a31779849f216a89f1b5 Author: Karsten Loesing Date: Fri Sep 27 10:06:13 2019 +0200 Use metrics-base ant task to fetch metrics-lib. Implements part of #31649. --- .gitlab-ci.yml | 7 +-- src/build | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b67b5a4..897c9da 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,5 @@ variables: GIT_STRATEGY: clone - METRICS_LIB_VERSION: "2.6.2" JAVA_TOOL_OPTIONS: "-Dfile.encoding=UTF-8" stages: @@ -15,11 +14,7 @@ test: - git submodule init - git submodule update - mkdir lib - - mkdir tmp - - pushd tmp - - curl https://dist.torproject.org/metrics-lib/$METRICS_LIB_VERSION/metrics-lib-$METRICS_LIB_VERSION.tar.gz | tar xzf - - - popd - - mv tmp/metrics-lib-$METRICS_LIB_VERSION/generated/dist/metrics-lib-$METRICS_LIB_VERSION-thin.jar lib + - ant fetch-metrics-lib - ant -lib /usr/share/java resolve - ant test - ant checks diff --git a/src/build b/src/build index 75cda35..07c2a00 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 75cda35ba7375ab0f5b12fa214620c3a29917097 +Subproject commit 07c2a00c27f0d536223f8b5a61fc91e60eb524d4 ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [exonerator/master] Use metrics-base ant task to fetch metrics-lib.
commit bc0c4db8ad9048d8f826b781274055ab9d475e9a Author: Karsten Loesing Date: Fri Sep 27 10:06:08 2019 +0200 Use metrics-base ant task to fetch metrics-lib. Implements part of #31649. --- .gitlab-ci.yml | 7 +-- src/build | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b67b5a4..897c9da 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,5 @@ variables: GIT_STRATEGY: clone - METRICS_LIB_VERSION: "2.6.2" JAVA_TOOL_OPTIONS: "-Dfile.encoding=UTF-8" stages: @@ -15,11 +14,7 @@ test: - git submodule init - git submodule update - mkdir lib - - mkdir tmp - - pushd tmp - - curl https://dist.torproject.org/metrics-lib/$METRICS_LIB_VERSION/metrics-lib-$METRICS_LIB_VERSION.tar.gz | tar xzf - - - popd - - mv tmp/metrics-lib-$METRICS_LIB_VERSION/generated/dist/metrics-lib-$METRICS_LIB_VERSION-thin.jar lib + - ant fetch-metrics-lib - ant -lib /usr/share/java resolve - ant test - ant checks diff --git a/src/build b/src/build index 0674958..07c2a00 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 0674958512c705255d4ec4631d3280068ce0a7af +Subproject commit 07c2a00c27f0d536223f8b5a61fc91e60eb524d4 ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/master] Use metrics-base ant task to fetch metrics-lib.
commit de03530b98bd66eee404e4483d930baf62a8f3e1 Author: Karsten Loesing Date: Fri Sep 27 10:05:35 2019 +0200 Use metrics-base ant task to fetch metrics-lib. Implements part of #31649. --- .gitlab-ci.yml | 7 +-- src/build | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5bd5623..897c9da 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,5 @@ variables: GIT_STRATEGY: clone - METRICS_LIB_VERSION: "2.7.0" JAVA_TOOL_OPTIONS: "-Dfile.encoding=UTF-8" stages: @@ -15,11 +14,7 @@ test: - git submodule init - git submodule update - mkdir lib - - mkdir tmp - - pushd tmp - - curl https://dist.torproject.org/metrics-lib/$METRICS_LIB_VERSION/metrics-lib-$METRICS_LIB_VERSION.tar.gz | tar xzf - - - popd - - mv tmp/metrics-lib-$METRICS_LIB_VERSION/generated/dist/metrics-lib-$METRICS_LIB_VERSION-thin.jar lib + - ant fetch-metrics-lib - ant -lib /usr/share/java resolve - ant test - ant checks diff --git a/src/build b/src/build index 0674958..07c2a00 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 0674958512c705255d4ec4631d3280068ce0a7af +Subproject commit 07c2a00c27f0d536223f8b5a61fc91e60eb524d4 ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/master] Use ant tasks to fetch metrics dependencies.
commit c30b25acbd32c5c6ff7c23c47062caf131d4a783
Author: Karsten Loesing
Date: Mon Sep 23 15:28:41 2019 +0200
Use ant tasks to fetch metrics dependencies.
Implements part of #31649.
---
.gitlab-ci.yml | 11 ++-
build.xml | 14 ++
src/build | 2 +-
src/submods/metrics-lib | 2 +-
4 files changed, 18 insertions(+), 11 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d10fcb8..ba2f6c1 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,7 +1,5 @@
variables:
GIT_STRATEGY: clone
- METRICS_LIB_VERSION: "2.6.2"
- EXONERATOR_VERSION: "4.1.0"
JAVA_TOOL_OPTIONS: "-Dfile.encoding=UTF-8"
stages:
@@ -16,13 +14,8 @@ test:
- git submodule init
- git submodule update
- mkdir lib
- - mkdir tmp
- - pushd tmp
- - curl
https://dist.torproject.org/metrics-lib/$METRICS_LIB_VERSION/metrics-lib-$METRICS_LIB_VERSION.tar.gz
| tar xzf -
- - curl
https://dist.torproject.org/exonerator/$EXONERATOR_VERSION/exonerator-$EXONERATOR_VERSION.tar.gz
| tar xzf -
- - popd
- - mv
tmp/metrics-lib-$METRICS_LIB_VERSION/generated/dist/metrics-lib-$METRICS_LIB_VERSION-thin.jar
lib
- - mv
tmp/exonerator-$EXONERATOR_VERSION/generated/dist/exonerator-$EXONERATOR_VERSION-thin.jar
lib
+ - ant fetch-metrics-lib
+ - ant fetch-exonerator
- ant -lib /usr/share/java resolve
- ant test
- ant checks
diff --git a/build.xml b/build.xml
index 3e8048f..959d069 100644
--- a/build.xml
+++ b/build.xml
@@ -308,6 +308,20 @@
+
+
+https://dist.torproject.org/exonerator/${exoneratorversion}/exonerator-${exoneratorversion}.tar.gz;
+ dest="${downloadedlibs}"/>
+
+
+
+
+
+
+
+
diff --git a/src/build b/src/build
index 0674958..07c2a00 16
--- a/src/build
+++ b/src/build
@@ -1 +1 @@
-Subproject commit 0674958512c705255d4ec4631d3280068ce0a7af
+Subproject commit 07c2a00c27f0d536223f8b5a61fc91e60eb524d4
diff --git a/src/submods/metrics-lib b/src/submods/metrics-lib
index c324d11..c1581dd 16
--- a/src/submods/metrics-lib
+++ b/src/submods/metrics-lib
@@ -1 +1 @@
-Subproject commit c324d11035059bd9677af120dc6471aebebf980a
+Subproject commit c1581dd8ca26798dc90ea2189ca41636b50e1e0d
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-lib/master] Upgrade to latest metrics-base.
commit c1581dd8ca26798dc90ea2189ca41636b50e1e0d Author: Karsten Loesing Date: Fri Sep 27 08:58:47 2019 +0200 Upgrade to latest metrics-base. --- src/build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build b/src/build index 0674958..07c2a00 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 0674958512c705255d4ec4631d3280068ce0a7af +Subproject commit 07c2a00c27f0d536223f8b5a61fc91e60eb524d4 ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-base/master] Add ant task to fetch metrics-lib from dist.
commit 07c2a00c27f0d536223f8b5a61fc91e60eb524d4
Author: Karsten Loesing
Date: Mon Sep 23 15:27:32 2019 +0200
Add ant task to fetch metrics-lib from dist.
Implements part of #31649.
---
java/base.xml | 16
1 file changed, 16 insertions(+)
diff --git a/java/base.xml b/java/base.xml
index 8342e91..4ce1660 100644
--- a/java/base.xml
+++ b/java/base.xml
@@ -28,6 +28,7 @@
+
@@ -99,6 +100,7 @@
+
@@ -109,6 +111,20 @@
+
+
+https://dist.torproject.org/metrics-lib/${metricslibversion}/metrics-lib-${metricslibversion}.tar.gz;
+ dest="${downloadedlibs}"/>
+
+
+
+
+
+
+
+
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [tech-reports/master] Don't assign the same report number twice.
commit ce8a199b104e3f300fad42be7733119a8a412b40
Author: Karsten Loesing
Date: Mon Sep 23 10:23:24 2019 +0200
Don't assign the same report number twice.
---
2018/side-channel-analysis/side-channel-analysis.tex | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/2018/side-channel-analysis/side-channel-analysis.tex
b/2018/side-channel-analysis/side-channel-analysis.tex
index 4824c3a..c712dad 100644
--- a/2018/side-channel-analysis/side-channel-analysis.tex
+++ b/2018/side-channel-analysis/side-channel-analysis.tex
@@ -3,7 +3,7 @@
\author{Nick Mathewson and Mike Perry}
\contact{\{nickm,mikeperry\}@torproject.org}
-\reportid{2018-11-001}
+\reportid{2018-11-002}
\date{November 27, 2018}
\title{Towards Side Channel Analysis of Datagram Tor vs Current Tor}
\maketitle
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [tech-reports/master] Convert Nick's and Mike's whitepaper to LaTeX.
commit 5e5007b4d9f7d974c5780fc9f94b5ca1e2c75737
Author: Karsten Loesing
Date: Mon Sep 16 10:41:12 2019 +0200
Convert Nick's and Mike's whitepaper to LaTeX.
---
...annel-analysis.md => side-channel-analysis.tex} | 122 +++--
1 file changed, 64 insertions(+), 58 deletions(-)
diff --git a/2018/side-channel-analysis/side-channel-analysis.md
b/2018/side-channel-analysis/side-channel-analysis.tex
similarity index 86%
rename from 2018/side-channel-analysis/side-channel-analysis.md
rename to 2018/side-channel-analysis/side-channel-analysis.tex
index f9454de..4824c3a 100644
--- a/2018/side-channel-analysis/side-channel-analysis.md
+++ b/2018/side-channel-analysis/side-channel-analysis.tex
@@ -1,17 +1,21 @@
-# Towards Side Channel Analysis of Datagram Tor vs Current Tor
+\documentclass{tortechrep}
+\begin{document}
- Version 0.6, 27 Nov 2018
+\author{Nick Mathewson and Mike Perry}
+\contact{\{nickm,mikeperry\}@torproject.org}
+\reportid{2018-11-001}
+\date{November 27, 2018}
+\title{Towards Side Channel Analysis of Datagram Tor vs Current Tor}
+\maketitle
- by Nick Mathewson and Mike Perry
-
-## Disclaimers
+\section{Disclaimers}
This whitepaper assumes that you know how Tor works.
There are probably some very good references here that we didn't
remember to cite.
-## Introduction
+\section{Introduction}
Tor's current design requires that its data cells be transmitted
from one end of a circuit to the other using a reliable, in-order
@@ -46,31 +50,33 @@ our description of the problem space will inspire, not
discourage,
future experiments in this area, and help with a holistic understanding
of the risks, rewards, and future areas of work.
-### A toy system
+\subsection{A toy system}
We will be analyzing a system that differs from Tor in the following
ways.
- * The link between a client and its guard, and between each pair
+\begin{itemize}
+\item The link between a client and its guard, and between each pair
of relays uses DTLS over UDP: packets can be dropped or
re-ordered by an attacker on the link, but not modified, read,
or forged. Each DTLS packet contains an integer number of cells.
- * Each circuit between a client and an exit traverse several
+\item Each circuit between a client and an exit traverse several
relays, as before. The cells on a circuit are no longer
guaranteed to arrive reliably, but can be dropped or re-ordered
on the wire, or by a relay.
- * To provide reliable service end-to-end, the client and the
+\item To provide reliable service end-to-end, the client and the
exit each use a TCP-like protocol to track which application
bytes have been sent and received. Received data is
acknowledged; dropped data is retransmitted.
- * The cryptography to be used for circuit encryption is not
+\item The cryptography to be used for circuit encryption is not
specified here.
- * A reliable signaling mechanism between relays (to create,
+\item A reliable signaling mechanism between relays (to create,
destroy, and maintain circuits) is not specified here.
+\end{itemize}
(It is likely that many readers will be able to design a system that
resists the attacks below better than the design above. But please
@@ -80,7 +86,7 @@ clearly superior to Tor as it is today. Before we can
deploy, we
will need not just defenses, but also a systemic way to compare the
effect of these defenses, used together, to the Tor status quo.)
-## Some preexisting attacks to consider
+\section{Some preexisting attacks to consider}
To put the datagram-based attacks into context, we'll start out by
listing some attacks against the current non-datagram Tor design
@@ -100,7 +106,7 @@ alter packet timing information in transit.
In many cases, we don't have good metrics or evaluation methodology
to determine how much harder or easier one attack is than another.
-### End-to-end passive traffic correlation attacks.
+\subsection{End-to-end passive traffic correlation attacks.}
Here's the gold-standard base-line attack: an attacker who can watch
any two points on the same circuit is assumed to be able to realize,
@@ -129,7 +135,7 @@ padding, it is not yet clear if we can use these defenses
in an
affordable way against a correlation attack, and it is hard to
measure their effectiveness on a realistic Tor-sized network.
-### Data tagging side-channels by relays
+\subsection{Data tagging side-channels by relays}
If two relays are on the same circuit, they can surreptitiously
communicate with one another transforming the data in the RELAY
@@ -148,7 +154,7 @@ To defend against this, we plan to replace our encryption
with a
non-malleable algorithm. See for example proposals 202, 261, and
295.
-### Destructive side-channels (internal)
+\subsection{Destructive side-channels (internal)}
Even if we remove the malleability in Tor's encryption, a smaller
s
[tor-commits] [tech-reports/master] Add Nick's and Mike's whitepaper draft.
commit 06ed620899c8e460369e21e15c5bdf9dc5a03e89 Author: Karsten Loesing Date: Mon Sep 16 10:11:24 2019 +0200 Add Nick's and Mike's whitepaper draft. --- .../side-channel-analysis/side-channel-analysis.md | 505 + 1 file changed, 505 insertions(+) diff --git a/2018/side-channel-analysis/side-channel-analysis.md b/2018/side-channel-analysis/side-channel-analysis.md new file mode 100644 index 000..f9454de --- /dev/null +++ b/2018/side-channel-analysis/side-channel-analysis.md @@ -0,0 +1,505 @@ +# Towards Side Channel Analysis of Datagram Tor vs Current Tor + + Version 0.6, 27 Nov 2018 + + by Nick Mathewson and Mike Perry + +## Disclaimers + +This whitepaper assumes that you know how Tor works. + +There are probably some very good references here that we didn't +remember to cite. + +## Introduction + +Tor's current design requires that its data cells be transmitted +from one end of a circuit to the other using a reliable, in-order +delivery mechanism. To meet this requirement, Tor relays need to +buffer cells--spending resources, hurting performance, and risking +susceptibility to out-of-memory attacks. + +In order to improve Tor's performance and resilience, researchers +have made several proposals for ways to relax the requirement for +reliable in-order delivery. In general, these "datagram-based" +proposals would allow relays to drop or reorder cells as needed, +and move the responsibility for providing a reliable stream protocol +to the endpoints (the client and the exit relays). + +But by increasing flexibility for the relays, and by increasing the +complexity of the endpoints, these datagram proposals also create +some new attack vectors. Before we can deploy any of these designs, +we need to consider whether these attacks weaken Tor's security, or +whether they are irrelevant given other, stronger attacks against +Onion Routing. + +This whitepaper tries to list these attacks, and to provide a +framework for thinking about them as we move forward with our design +analysis. + +We hope that this whitepaper will help researchers and others in the Tor +community to understand these issues, so that we can work together to +find new ideas to analyze and mitigate the attacks described here, and +to help deploy a faster and more reliable network while still +maintaining our current (or better) security guarantees. We hope that +our description of the problem space will inspire, not discourage, +future experiments in this area, and help with a holistic understanding +of the risks, rewards, and future areas of work. + +### A toy system + +We will be analyzing a system that differs from Tor in the following +ways. + + * The link between a client and its guard, and between each pair +of relays uses DTLS over UDP: packets can be dropped or +re-ordered by an attacker on the link, but not modified, read, +or forged. Each DTLS packet contains an integer number of cells. + + * Each circuit between a client and an exit traverse several +relays, as before. The cells on a circuit are no longer +guaranteed to arrive reliably, but can be dropped or re-ordered +on the wire, or by a relay. + + * To provide reliable service end-to-end, the client and the +exit each use a TCP-like protocol to track which application +bytes have been sent and received. Received data is +acknowledged; dropped data is retransmitted. + + * The cryptography to be used for circuit encryption is not +specified here. + + * A reliable signaling mechanism between relays (to create, +destroy, and maintain circuits) is not specified here. + +(It is likely that many readers will be able to design a system that +resists the attacks below better than the design above. But please +remember as you do, that a design which improves a system in one way +may constrain it in others, or may offer insufficient benefits to be +clearly superior to Tor as it is today. Before we can deploy, we +will need not just defenses, but also a systemic way to compare the +effect of these defenses, used together, to the Tor status quo.) + +## Some preexisting attacks to consider + +To put the datagram-based attacks into context, we'll start out by +listing some attacks against the current non-datagram Tor design +(and proposed defenses for those, where they exist). + +We assume, as usual, an adversary who controls some but not all +relays, and some but not all ISPs. + +A note on attack power: the accuracy of many of these attacks, +particularly the passive ones, depends on the type of traffic being +sent, the quantity of similar traffic elsewhere on the Tor network, +the quantity of concurrent activity by the same client, the +adversary's observation position and data retention resolution, the +quantity of padding, and the tendency of the network to preserve or +alter packet timing information in transit. + +In many cases, we don't have go
[tor-commits] [tech-reports/master] Also add template and .gitignore for Nick's/Nike's report.
commit 508515c916237001b209b320d276e50020ef06f2 Author: Karsten Loesing Date: Mon Sep 23 09:52:40 2019 +0200 Also add template and .gitignore for Nick's/Nike's report. --- 2018/side-channel-analysis/.gitignore | 2 ++ 2018/side-channel-analysis/tortechrep.cls | 1 + 2 files changed, 3 insertions(+) diff --git a/2018/side-channel-analysis/.gitignore b/2018/side-channel-analysis/.gitignore new file mode 100644 index 000..e10caac --- /dev/null +++ b/2018/side-channel-analysis/.gitignore @@ -0,0 +1,2 @@ +side-channel-analysis-2018-11-27.pdf + diff --git a/2018/side-channel-analysis/tortechrep.cls b/2018/side-channel-analysis/tortechrep.cls new file mode 12 index 000..4c24db2 --- /dev/null +++ b/2018/side-channel-analysis/tortechrep.cls @@ -0,0 +1 @@ +../../tortechrep.cls \ No newline at end of file ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/master] Re-add recent bridge pool assignments.
commit 7ccae10c5c171d363d907dd74736d51d89725748 Author: Karsten Loesing Date: Thu Sep 19 16:30:16 2019 +0200 Re-add recent bridge pool assignments. --- src/main/resources/web/jsps/collector.jsp | 9 +++-- 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/main/resources/web/jsps/collector.jsp b/src/main/resources/web/jsps/collector.jsp index 42280a9..102bcab 100644 --- a/src/main/resources/web/jsps/collector.jsp +++ b/src/main/resources/web/jsps/collector.jsp @@ -158,7 +158,8 @@ Bridge Pool Assignments @type bridge-pool-assignment 1.0 - archive + recent + archive TorDNSEL's Exit Lists @@ -573,6 +574,7 @@ statistical analysis. Bridge Pool Assignments @type bridge-pool-assignment 1.0 + recent archive # @@ -613,11 +615,6 @@ bridge-pool-assignment 2011-03-13 14:38:03 [...] - -As of December 8, 2014, bridge pool assignment files are no longer -archived. - - TorDNSEL's Exit Lists # ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/master] Bump version to 1.11.1-dev.
commit 18877fdb4d270a6ef02664eb07271c064fab785f Author: Karsten Loesing Date: Thu Sep 19 15:11:37 2019 +0200 Bump version to 1.11.1-dev. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index 9f2a0a6..6b097a3 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,7 @@ - + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/release] Fix bridge pool assignments sync and log statement.
commit 8ed8de6e76e0f45f6112f3d60597037bdf27493f
Author: Karsten Loesing
Date: Thu Sep 19 14:57:15 2019 +0200
Fix bridge pool assignments sync and log statement.
---
CHANGELOG.md | 2 ++
.../collector/bridgepools/BridgePoolAssignmentsProcessor.java| 5 -
2 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d6992c9..48eae33 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,8 @@
* Minor changes
- Update create-tarballs.sh to also produce bridge pool assignments
tarballs.
+ - Fix synchronization part of newly re-added bridge pool
+ assignments module.
# Changes in version 1.11.0 - 2019-09-19
diff --git
a/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java
b/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java
index a386fb6..875273d 100644
---
a/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java
+++
b/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java
@@ -3,6 +3,7 @@
package org.torproject.metrics.collector.bridgepools;
+import org.torproject.descriptor.BridgePoolAssignment;
import org.torproject.metrics.collector.conf.Configuration;
import org.torproject.metrics.collector.conf.ConfigurationException;
import org.torproject.metrics.collector.conf.Key;
@@ -79,6 +80,8 @@ public class BridgePoolAssignmentsProcessor extends
CollecTorMain {
*/
public BridgePoolAssignmentsProcessor(Configuration config) {
super(config);
+this.mapPathDescriptors.put("recent/bridge-pool-assignments",
+BridgePoolAssignment.class);
}
/**
@@ -153,7 +156,7 @@ public class BridgePoolAssignmentsProcessor extends
CollecTorMain {
}
}
if (null != latestPublished
-&& latestPublished.minusMinutes(330L).isBefore(LocalDateTime.now())) {
+&& latestPublished.plusMinutes(330L).isBefore(LocalDateTime.now())) {
logger.warn("The last known bridge pool assignment list was "
+ "published at {}, which is more than 5:30 hours in the past.",
latestPublished);
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/release] Update create-tarballs.sh script and PROTOCOL.
commit d3203ceb9e726d641ea1a2621ff325873b7d6d45
Author: Karsten Loesing
Date: Thu Sep 19 12:37:18 2019 +0200
Update create-tarballs.sh script and PROTOCOL.
These changes still belong to archiving bridge pool assignments. But
they're not important enough for putting out a new release at this
point.
---
CHANGELOG.md | 7 +++
src/main/resources/create-tarballs.sh | 7 +++
src/main/resources/docs/PROTOCOL | 5 +
3 files changed, 15 insertions(+), 4 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f5f9488..d6992c9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,10 @@
+# Changes in version 1.??.? - 2019-??-??
+
+ * Minor changes
+ - Update create-tarballs.sh to also produce bridge pool assignments
+ tarballs.
+
+
# Changes in version 1.11.0 - 2019-09-19
* Medium changes
diff --git a/src/main/resources/create-tarballs.sh
b/src/main/resources/create-tarballs.sh
index 50b7fdb..5802020 100755
--- a/src/main/resources/create-tarballs.sh
+++ b/src/main/resources/create-tarballs.sh
@@ -61,6 +61,8 @@ TARBALLS=(
bridge-extra-infos-$YEARTWO-$MONTHTWO
snowflakes-$YEARONE-$MONTHONE
snowflakes-$YEARTWO-$MONTHTWO
+ bridge-pool-assignments-$YEARONE-$MONTHONE
+ bridge-pool-assignments-$YEARTWO-$MONTHTWO
)
TARBALLS=($(printf "%s\n" "${TARBALLS[@]}" | uniq))
@@ -90,6 +92,8 @@ DIRECTORIES=(
$OUTDIR/bridge-descriptors/$YEARTWO/$MONTHTWO/extra-infos/
$OUTDIR/snowflakes/$YEARONE/$MONTHONE/
$OUTDIR/snowflakes/$YEARTWO/$MONTHTWO/
+ $OUTDIR/bridge-pool-assignments/$YEARONE/$MONTHONE/
+ $OUTDIR/bridge-pool-assignments/$YEARTWO/$MONTHTWO/
)
DIRECTORIES=($(printf "%s\n" "${DIRECTORIES[@]}" | uniq))
@@ -176,4 +180,7 @@ ln -f -s -t $ARCHIVEDIR/webstats/
$TARBALLTARGETDIR/webstats-20??-??.tar
mkdir -p $ARCHIVEDIR/snowflakes/
ln -f -s -t $ARCHIVEDIR/snowflakes/ $TARBALLTARGETDIR/snowflakes-20??-??.tar.xz
+mkdir -p $ARCHIVEDIR/bridge-pool-assignments/
+ln -f -s -t $ARCHIVEDIR/bridge-pool-assignments/
$TARBALLTARGETDIR/bridge-pool-assignments-20??-??.tar.xz
+
echo `date` "Finished."
diff --git a/src/main/resources/docs/PROTOCOL b/src/main/resources/docs/PROTOCOL
index 478f168..d735d40 100644
--- a/src/main/resources/docs/PROTOCOL
+++ b/src/main/resources/docs/PROTOCOL
@@ -282,15 +282,12 @@
The top level of subdirectories is
* bridge-descriptors
+ * bridge-pool-assignments
* exit-lists
* relay-descriptors
* torperf
* snowflakes
- (There has been another subdirectory 'bridge-pool-assignments' which has
been
- removed when CollecTor stopped collecting those descriptors. However, it's
- structure can still be found in the tarballs.)
-
5.1 'exit-lists' and 'torperf' Tars
5.1.1
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/release] Prepare for 1.11.1 release.
commit 91e5a07df83f4432043afe13ad221d8d8992c176 Author: Karsten Loesing Date: Thu Sep 19 15:00:00 2019 +0200 Prepare for 1.11.1 release. --- CHANGELOG.md | 2 +- build.xml| 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48eae33..e30b086 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changes in version 1.??.? - 2019-??-?? +# Changes in version 1.11.1 - 2019-09-19 * Minor changes - Update create-tarballs.sh to also produce bridge pool assignments diff --git a/build.xml b/build.xml index da10311..9f2a0a6 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,7 @@ - + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/release] Bump version to 1.11.0-dev.
commit cb6e85d79a8622248a68166ae9cd87febcbd0b45 Author: Karsten Loesing Date: Thu Sep 19 11:32:29 2019 +0200 Bump version to 1.11.0-dev. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index faf44a9..da10311 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,7 @@ - + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/master] Fix bridge pool assignments sync and log statement.
commit 8ed8de6e76e0f45f6112f3d60597037bdf27493f
Author: Karsten Loesing
Date: Thu Sep 19 14:57:15 2019 +0200
Fix bridge pool assignments sync and log statement.
---
CHANGELOG.md | 2 ++
.../collector/bridgepools/BridgePoolAssignmentsProcessor.java| 5 -
2 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d6992c9..48eae33 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,8 @@
* Minor changes
- Update create-tarballs.sh to also produce bridge pool assignments
tarballs.
+ - Fix synchronization part of newly re-added bridge pool
+ assignments module.
# Changes in version 1.11.0 - 2019-09-19
diff --git
a/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java
b/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java
index a386fb6..875273d 100644
---
a/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java
+++
b/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java
@@ -3,6 +3,7 @@
package org.torproject.metrics.collector.bridgepools;
+import org.torproject.descriptor.BridgePoolAssignment;
import org.torproject.metrics.collector.conf.Configuration;
import org.torproject.metrics.collector.conf.ConfigurationException;
import org.torproject.metrics.collector.conf.Key;
@@ -79,6 +80,8 @@ public class BridgePoolAssignmentsProcessor extends
CollecTorMain {
*/
public BridgePoolAssignmentsProcessor(Configuration config) {
super(config);
+this.mapPathDescriptors.put("recent/bridge-pool-assignments",
+BridgePoolAssignment.class);
}
/**
@@ -153,7 +156,7 @@ public class BridgePoolAssignmentsProcessor extends
CollecTorMain {
}
}
if (null != latestPublished
-&& latestPublished.minusMinutes(330L).isBefore(LocalDateTime.now())) {
+&& latestPublished.plusMinutes(330L).isBefore(LocalDateTime.now())) {
logger.warn("The last known bridge pool assignment list was "
+ "published at {}, which is more than 5:30 hours in the past.",
latestPublished);
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/master] Prepare for 1.11.1 release.
commit 91e5a07df83f4432043afe13ad221d8d8992c176 Author: Karsten Loesing Date: Thu Sep 19 15:00:00 2019 +0200 Prepare for 1.11.1 release. --- CHANGELOG.md | 2 +- build.xml| 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48eae33..e30b086 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changes in version 1.??.? - 2019-??-?? +# Changes in version 1.11.1 - 2019-09-19 * Minor changes - Update create-tarballs.sh to also produce bridge pool assignments diff --git a/build.xml b/build.xml index da10311..9f2a0a6 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,7 @@ - + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/master] Update create-tarballs.sh script and PROTOCOL.
commit d3203ceb9e726d641ea1a2621ff325873b7d6d45
Author: Karsten Loesing
Date: Thu Sep 19 12:37:18 2019 +0200
Update create-tarballs.sh script and PROTOCOL.
These changes still belong to archiving bridge pool assignments. But
they're not important enough for putting out a new release at this
point.
---
CHANGELOG.md | 7 +++
src/main/resources/create-tarballs.sh | 7 +++
src/main/resources/docs/PROTOCOL | 5 +
3 files changed, 15 insertions(+), 4 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f5f9488..d6992c9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,10 @@
+# Changes in version 1.??.? - 2019-??-??
+
+ * Minor changes
+ - Update create-tarballs.sh to also produce bridge pool assignments
+ tarballs.
+
+
# Changes in version 1.11.0 - 2019-09-19
* Medium changes
diff --git a/src/main/resources/create-tarballs.sh
b/src/main/resources/create-tarballs.sh
index 50b7fdb..5802020 100755
--- a/src/main/resources/create-tarballs.sh
+++ b/src/main/resources/create-tarballs.sh
@@ -61,6 +61,8 @@ TARBALLS=(
bridge-extra-infos-$YEARTWO-$MONTHTWO
snowflakes-$YEARONE-$MONTHONE
snowflakes-$YEARTWO-$MONTHTWO
+ bridge-pool-assignments-$YEARONE-$MONTHONE
+ bridge-pool-assignments-$YEARTWO-$MONTHTWO
)
TARBALLS=($(printf "%s\n" "${TARBALLS[@]}" | uniq))
@@ -90,6 +92,8 @@ DIRECTORIES=(
$OUTDIR/bridge-descriptors/$YEARTWO/$MONTHTWO/extra-infos/
$OUTDIR/snowflakes/$YEARONE/$MONTHONE/
$OUTDIR/snowflakes/$YEARTWO/$MONTHTWO/
+ $OUTDIR/bridge-pool-assignments/$YEARONE/$MONTHONE/
+ $OUTDIR/bridge-pool-assignments/$YEARTWO/$MONTHTWO/
)
DIRECTORIES=($(printf "%s\n" "${DIRECTORIES[@]}" | uniq))
@@ -176,4 +180,7 @@ ln -f -s -t $ARCHIVEDIR/webstats/
$TARBALLTARGETDIR/webstats-20??-??.tar
mkdir -p $ARCHIVEDIR/snowflakes/
ln -f -s -t $ARCHIVEDIR/snowflakes/ $TARBALLTARGETDIR/snowflakes-20??-??.tar.xz
+mkdir -p $ARCHIVEDIR/bridge-pool-assignments/
+ln -f -s -t $ARCHIVEDIR/bridge-pool-assignments/
$TARBALLTARGETDIR/bridge-pool-assignments-20??-??.tar.xz
+
echo `date` "Finished."
diff --git a/src/main/resources/docs/PROTOCOL b/src/main/resources/docs/PROTOCOL
index 478f168..d735d40 100644
--- a/src/main/resources/docs/PROTOCOL
+++ b/src/main/resources/docs/PROTOCOL
@@ -282,15 +282,12 @@
The top level of subdirectories is
* bridge-descriptors
+ * bridge-pool-assignments
* exit-lists
* relay-descriptors
* torperf
* snowflakes
- (There has been another subdirectory 'bridge-pool-assignments' which has
been
- removed when CollecTor stopped collecting those descriptors. However, it's
- structure can still be found in the tarballs.)
-
5.1 'exit-lists' and 'torperf' Tars
5.1.1
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/master] Bump version to 1.11.0-dev.
commit cb6e85d79a8622248a68166ae9cd87febcbd0b45 Author: Karsten Loesing Date: Thu Sep 19 11:32:29 2019 +0200 Bump version to 1.11.0-dev. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index faf44a9..da10311 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,7 @@ - + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/release] Prepare for 1.11.0 release.
commit a51b6e49cde838422f44d54881d972f17f28372a Author: Karsten Loesing Date: Thu Sep 19 11:10:49 2019 +0200 Prepare for 1.11.0 release. --- CHANGELOG.md | 7 ++- build.xml| 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ecfb35..f5f9488 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# Changes in version 1.11.0 - 2019-09-19 + + * Medium changes + - Archive bridge pool assignments again. + + # Changes in version 1.10.0 - 2019-09-12 * Medium changes @@ -15,7 +21,6 @@ - Remove Cobertura from the build process. - Archive snowflake statistics. - Update to metrics-lib 2.7.0. - - Archive bridge pool assignments again. # Changes in version 1.9.1 - 2019-05-29 diff --git a/build.xml b/build.xml index 4bada49..faf44a9 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,7 @@ - + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/release] Bump version to 1.10.0-dev.
commit 16894a4d0096ac76e0ccbba1cd7b593ab1ddab2c Author: Karsten Loesing Date: Thu Sep 12 10:59:05 2019 +0200 Bump version to 1.10.0-dev. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index f4812c4..4bada49 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,7 @@ - + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/release] Archive bridge pool assignments again.
commit b2b3363232fd1a425a6e83bde89b2687b56abc0a
Author: Karsten Loesing
Date: Fri Aug 30 11:09:23 2019 +0200
Archive bridge pool assignments again.
Implements #31558.
---
CHANGELOG.md | 1 +
.../org/torproject/metrics/collector/Main.java | 3 +
.../BridgePoolAssignmentsProcessor.java| 363 +
.../metrics/collector/conf/Annotation.java | 1 +
.../metrics/collector/conf/Configuration.java | 1 +
.../org/torproject/metrics/collector/conf/Key.java | 6 +
.../persist/BridgePoolAssignmentPersistence.java | 34 ++
.../collector/persist/DescriptorPersistence.java | 2 +
.../metrics/collector/sync/SyncPersistence.java| 6 +
src/main/resources/collector.properties| 18 +
.../metrics/collector/conf/ConfigurationTest.java | 2 +-
.../metrics/collector/cron/CollecTorMainTest.java | 1 +
12 files changed, 437 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b38b124..4ecfb35 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,7 @@
- Remove Cobertura from the build process.
- Archive snowflake statistics.
- Update to metrics-lib 2.7.0.
+ - Archive bridge pool assignments again.
# Changes in version 1.9.1 - 2019-05-29
diff --git a/src/main/java/org/torproject/metrics/collector/Main.java
b/src/main/java/org/torproject/metrics/collector/Main.java
index 6907e93..3150ffc 100644
--- a/src/main/java/org/torproject/metrics/collector/Main.java
+++ b/src/main/java/org/torproject/metrics/collector/Main.java
@@ -4,6 +4,7 @@
package org.torproject.metrics.collector;
import org.torproject.metrics.collector.bridgedescs.SanitizedBridgesWriter;
+import
org.torproject.metrics.collector.bridgepools.BridgePoolAssignmentsProcessor;
import org.torproject.metrics.collector.conf.Configuration;
import org.torproject.metrics.collector.conf.ConfigurationException;
import org.torproject.metrics.collector.conf.Key;
@@ -49,6 +50,8 @@ public class Main {
static { // add a new main class here
collecTorMains.put(Key.BridgedescsActivated, SanitizedBridgesWriter.class);
+collecTorMains.put(Key.BridgePoolAssignmentsActivated,
+BridgePoolAssignmentsProcessor.class);
collecTorMains.put(Key.ExitlistsActivated, ExitListDownloader.class);
collecTorMains.put(Key.UpdateindexActivated, CreateIndexJson.class);
collecTorMains.put(Key.RelaydescsActivated, ArchiveWriter.class);
diff --git
a/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java
b/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java
new file mode 100644
index 000..cad91ef
--- /dev/null
+++
b/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java
@@ -0,0 +1,363 @@
+/* Copyright 2011--2019 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.metrics.collector.bridgepools;
+
+import org.torproject.metrics.collector.conf.Configuration;
+import org.torproject.metrics.collector.conf.ConfigurationException;
+import org.torproject.metrics.collector.conf.Key;
+import org.torproject.metrics.collector.cron.CollecTorMain;
+
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.time.DateTimeException;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.Stack;
+import java.util.TreeMap;
+
+public class BridgePoolAssignmentsProcessor extends CollecTorMain {
+
+ /**
+ * Class logger.
+ */
+ private static final Logger logger = LoggerFactory.getLogger(
+ BridgePoolAssignmentsProcessor.class);
+
+ /**
+ * Directory containing original, not-yet-sanitized bridge pool assignment
+ * files.
+ */
+ private File assignmentsDirectory;
+
+ /**
+ * Directory containing sanitized bridge pool assignments for tarballs.
+ */
+ private String outputPathName;
+
+ /**
+ * Directory containing recently stored sanitized bridge pool assignments.
+ */
+ private String recentPathName;
+
+ /**
+ * Timestamp format in bridge-pool-assignments line.
+ */
+ private DateTimeFormatter assignmentFormat = DateTimeFormatter.ofPattern
[tor-commits] [collector/release] Tone down a warning about outdated assignments.
commit 54ac654d40544bab48cec58ba64546ef6ffe0bfe
Author: Karsten Loesing
Date: Thu Sep 19 11:08:41 2019 +0200
Tone down a warning about outdated assignments.
---
.../BridgePoolAssignmentsProcessor.java| 23 ++
1 file changed, 15 insertions(+), 8 deletions(-)
diff --git
a/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java
b/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java
index cad91ef..a386fb6 100644
---
a/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java
+++
b/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java
@@ -115,11 +115,19 @@ public class BridgePoolAssignmentsProcessor extends
CollecTorMain {
logger.info("Starting bridge-pool-assignments module of CollecTor.");
this.initializeConfiguration();
List assignmentFiles = this.listAssignmentFiles();
+LocalDateTime latestPublished = null;
for (File assignmentFile : assignmentFiles) {
logger.info("Processing bridge pool assignment file '{}'...",
assignmentFile.getAbsolutePath());
+ SortedMap>
+ readBridgePoolAssignments
+ = this.readBridgePoolAssignments(assignmentFile);
+ if (null == latestPublished
+ || readBridgePoolAssignments.lastKey().isAfter(latestPublished)) {
+latestPublished = readBridgePoolAssignments.lastKey();
+ }
for (Map.Entry> e
- : this.readBridgePoolAssignments(assignmentFile).entrySet()) {
+ : readBridgePoolAssignments.entrySet()) {
LocalDateTime published = e.getKey();
SortedMap originalAssignments = e.getValue();
SortedMap sanitizedAssignments
@@ -144,6 +152,12 @@ public class BridgePoolAssignmentsProcessor extends
CollecTorMain {
}
}
}
+if (null != latestPublished
+&& latestPublished.minusMinutes(330L).isBefore(LocalDateTime.now())) {
+ logger.warn("The last known bridge pool assignment list was "
+ + "published at {}, which is more than 5:30 hours in the past.",
+ latestPublished);
+}
this.cleanUpRsyncDirectory();
logger.info("Finished processing bridge pool assignment file(s).");
}
@@ -251,13 +265,6 @@ public class BridgePoolAssignmentsProcessor extends
CollecTorMain {
logger.warn("Could not read bridge pool assignment file '{}'. "
+ "Skipping.", assignmentFile.getAbsolutePath(), e);
}
-if (!readBridgePoolAssignments.isEmpty()
-&& readBridgePoolAssignments.lastKey().minusMinutes(330L)
-.isBefore(LocalDateTime.now())) {
- logger.warn("The last known bridge pool assignment list was "
- + "published at {}, which is more than 5:30 hours in the past.",
- readBridgePoolAssignments.lastKey());
-}
return readBridgePoolAssignments;
}
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/release] Upgrade to latest metrics-base.
commit 522a54ed4751faf9b595100ac63dd5fd0b446f25 Author: Karsten Loesing Date: Thu Sep 19 10:29:27 2019 +0200 Upgrade to latest metrics-base. --- src/build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build b/src/build index 75cda35..0674958 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 75cda35ba7375ab0f5b12fa214620c3a29917097 +Subproject commit 0674958512c705255d4ec4631d3280068ce0a7af ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/master] Tone down a warning about outdated assignments.
commit 54ac654d40544bab48cec58ba64546ef6ffe0bfe
Author: Karsten Loesing
Date: Thu Sep 19 11:08:41 2019 +0200
Tone down a warning about outdated assignments.
---
.../BridgePoolAssignmentsProcessor.java| 23 ++
1 file changed, 15 insertions(+), 8 deletions(-)
diff --git
a/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java
b/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java
index cad91ef..a386fb6 100644
---
a/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java
+++
b/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java
@@ -115,11 +115,19 @@ public class BridgePoolAssignmentsProcessor extends
CollecTorMain {
logger.info("Starting bridge-pool-assignments module of CollecTor.");
this.initializeConfiguration();
List assignmentFiles = this.listAssignmentFiles();
+LocalDateTime latestPublished = null;
for (File assignmentFile : assignmentFiles) {
logger.info("Processing bridge pool assignment file '{}'...",
assignmentFile.getAbsolutePath());
+ SortedMap>
+ readBridgePoolAssignments
+ = this.readBridgePoolAssignments(assignmentFile);
+ if (null == latestPublished
+ || readBridgePoolAssignments.lastKey().isAfter(latestPublished)) {
+latestPublished = readBridgePoolAssignments.lastKey();
+ }
for (Map.Entry> e
- : this.readBridgePoolAssignments(assignmentFile).entrySet()) {
+ : readBridgePoolAssignments.entrySet()) {
LocalDateTime published = e.getKey();
SortedMap originalAssignments = e.getValue();
SortedMap sanitizedAssignments
@@ -144,6 +152,12 @@ public class BridgePoolAssignmentsProcessor extends
CollecTorMain {
}
}
}
+if (null != latestPublished
+&& latestPublished.minusMinutes(330L).isBefore(LocalDateTime.now())) {
+ logger.warn("The last known bridge pool assignment list was "
+ + "published at {}, which is more than 5:30 hours in the past.",
+ latestPublished);
+}
this.cleanUpRsyncDirectory();
logger.info("Finished processing bridge pool assignment file(s).");
}
@@ -251,13 +265,6 @@ public class BridgePoolAssignmentsProcessor extends
CollecTorMain {
logger.warn("Could not read bridge pool assignment file '{}'. "
+ "Skipping.", assignmentFile.getAbsolutePath(), e);
}
-if (!readBridgePoolAssignments.isEmpty()
-&& readBridgePoolAssignments.lastKey().minusMinutes(330L)
-.isBefore(LocalDateTime.now())) {
- logger.warn("The last known bridge pool assignment list was "
- + "published at {}, which is more than 5:30 hours in the past.",
- readBridgePoolAssignments.lastKey());
-}
return readBridgePoolAssignments;
}
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/master] Prepare for 1.11.0 release.
commit a51b6e49cde838422f44d54881d972f17f28372a Author: Karsten Loesing Date: Thu Sep 19 11:10:49 2019 +0200 Prepare for 1.11.0 release. --- CHANGELOG.md | 7 ++- build.xml| 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ecfb35..f5f9488 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# Changes in version 1.11.0 - 2019-09-19 + + * Medium changes + - Archive bridge pool assignments again. + + # Changes in version 1.10.0 - 2019-09-12 * Medium changes @@ -15,7 +21,6 @@ - Remove Cobertura from the build process. - Archive snowflake statistics. - Update to metrics-lib 2.7.0. - - Archive bridge pool assignments again. # Changes in version 1.9.1 - 2019-05-29 diff --git a/build.xml b/build.xml index 4bada49..faf44a9 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,7 @@ - + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/master] Archive bridge pool assignments again.
commit b2b3363232fd1a425a6e83bde89b2687b56abc0a
Author: Karsten Loesing
Date: Fri Aug 30 11:09:23 2019 +0200
Archive bridge pool assignments again.
Implements #31558.
---
CHANGELOG.md | 1 +
.../org/torproject/metrics/collector/Main.java | 3 +
.../BridgePoolAssignmentsProcessor.java| 363 +
.../metrics/collector/conf/Annotation.java | 1 +
.../metrics/collector/conf/Configuration.java | 1 +
.../org/torproject/metrics/collector/conf/Key.java | 6 +
.../persist/BridgePoolAssignmentPersistence.java | 34 ++
.../collector/persist/DescriptorPersistence.java | 2 +
.../metrics/collector/sync/SyncPersistence.java| 6 +
src/main/resources/collector.properties| 18 +
.../metrics/collector/conf/ConfigurationTest.java | 2 +-
.../metrics/collector/cron/CollecTorMainTest.java | 1 +
12 files changed, 437 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b38b124..4ecfb35 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,7 @@
- Remove Cobertura from the build process.
- Archive snowflake statistics.
- Update to metrics-lib 2.7.0.
+ - Archive bridge pool assignments again.
# Changes in version 1.9.1 - 2019-05-29
diff --git a/src/main/java/org/torproject/metrics/collector/Main.java
b/src/main/java/org/torproject/metrics/collector/Main.java
index 6907e93..3150ffc 100644
--- a/src/main/java/org/torproject/metrics/collector/Main.java
+++ b/src/main/java/org/torproject/metrics/collector/Main.java
@@ -4,6 +4,7 @@
package org.torproject.metrics.collector;
import org.torproject.metrics.collector.bridgedescs.SanitizedBridgesWriter;
+import
org.torproject.metrics.collector.bridgepools.BridgePoolAssignmentsProcessor;
import org.torproject.metrics.collector.conf.Configuration;
import org.torproject.metrics.collector.conf.ConfigurationException;
import org.torproject.metrics.collector.conf.Key;
@@ -49,6 +50,8 @@ public class Main {
static { // add a new main class here
collecTorMains.put(Key.BridgedescsActivated, SanitizedBridgesWriter.class);
+collecTorMains.put(Key.BridgePoolAssignmentsActivated,
+BridgePoolAssignmentsProcessor.class);
collecTorMains.put(Key.ExitlistsActivated, ExitListDownloader.class);
collecTorMains.put(Key.UpdateindexActivated, CreateIndexJson.class);
collecTorMains.put(Key.RelaydescsActivated, ArchiveWriter.class);
diff --git
a/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java
b/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java
new file mode 100644
index 000..cad91ef
--- /dev/null
+++
b/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java
@@ -0,0 +1,363 @@
+/* Copyright 2011--2019 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.metrics.collector.bridgepools;
+
+import org.torproject.metrics.collector.conf.Configuration;
+import org.torproject.metrics.collector.conf.ConfigurationException;
+import org.torproject.metrics.collector.conf.Key;
+import org.torproject.metrics.collector.cron.CollecTorMain;
+
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.time.DateTimeException;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.Stack;
+import java.util.TreeMap;
+
+public class BridgePoolAssignmentsProcessor extends CollecTorMain {
+
+ /**
+ * Class logger.
+ */
+ private static final Logger logger = LoggerFactory.getLogger(
+ BridgePoolAssignmentsProcessor.class);
+
+ /**
+ * Directory containing original, not-yet-sanitized bridge pool assignment
+ * files.
+ */
+ private File assignmentsDirectory;
+
+ /**
+ * Directory containing sanitized bridge pool assignments for tarballs.
+ */
+ private String outputPathName;
+
+ /**
+ * Directory containing recently stored sanitized bridge pool assignments.
+ */
+ private String recentPathName;
+
+ /**
+ * Timestamp format in bridge-pool-assignments line.
+ */
+ private DateTimeFormatter assignmentFormat = DateTimeFormatter.ofPattern
[tor-commits] [collector/master] Upgrade to latest metrics-base.
commit 522a54ed4751faf9b595100ac63dd5fd0b446f25 Author: Karsten Loesing Date: Thu Sep 19 10:29:27 2019 +0200 Upgrade to latest metrics-base. --- src/build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build b/src/build index 75cda35..0674958 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 75cda35ba7375ab0f5b12fa214620c3a29917097 +Subproject commit 0674958512c705255d4ec4631d3280068ce0a7af ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [exonerator/master] Update PostgreSQL JDBC driver version to 42.2.5.
commit 6babf3f62e7b23beed348d27b25e897936f96b72 Author: Karsten Loesing Date: Tue Sep 17 10:27:28 2019 +0200 Update PostgreSQL JDBC driver version to 42.2.5. --- CHANGELOG.md | 1 + build.xml| 2 +- src/build| 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f0db8f..ddbae79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ versions resolved by Ivy are the same as in Debian stretch with few exceptions. - Remove Cobertura from the build process. + - Update PostgreSQL JDBC driver version to 42.2.5. # Changes in version 4.1.0 - 2019-05-13 diff --git a/build.xml b/build.xml index 9aab930..9d522fe 100644 --- a/build.xml +++ b/build.xml @@ -40,7 +40,7 @@ - + diff --git a/src/build b/src/build index 75cda35..0674958 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 75cda35ba7375ab0f5b12fa214620c3a29917097 +Subproject commit 0674958512c705255d4ec4631d3280068ce0a7af ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/master] Update to latest metrics-base and metrics-lib.
commit 3a663e8ff6b5b0c0757fe787cf92eaac91734ad5 Author: Karsten Loesing Date: Tue Sep 17 10:05:27 2019 +0200 Update to latest metrics-base and metrics-lib. This update includes an update of PostgreSQL JDBC driver version to 42.2.5. --- CHANGELOG.md| 1 + build.xml | 2 +- src/build | 2 +- src/submods/metrics-lib | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41bb5df..2147ed7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ versions resolved by Ivy are the same as in Debian stretch with few exceptions. - Remove Cobertura from the build process. + - Update PostgreSQL JDBC driver version to 42.2.5. # Changes in version 1.2.0 - 2018-08-25 diff --git a/build.xml b/build.xml index 038a82f..3e8048f 100644 --- a/build.xml +++ b/build.xml @@ -57,7 +57,7 @@ - + diff --git a/src/build b/src/build index 75cda35..0674958 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 75cda35ba7375ab0f5b12fa214620c3a29917097 +Subproject commit 0674958512c705255d4ec4631d3280068ce0a7af diff --git a/src/submods/metrics-lib b/src/submods/metrics-lib index 7e396f0..c324d11 16 --- a/src/submods/metrics-lib +++ b/src/submods/metrics-lib @@ -1 +1 @@ -Subproject commit 7e396f01ef6de4cb3cd7e2b7fc977b4af7b97e7f +Subproject commit c324d11035059bd9677af120dc6471aebebf980a ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-lib/master] Upgrade to latest metrics-base.
commit c324d11035059bd9677af120dc6471aebebf980a Author: Karsten Loesing Date: Tue Sep 17 10:00:00 2019 +0200 Upgrade to latest metrics-base. --- src/build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build b/src/build index 75cda35..0674958 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 75cda35ba7375ab0f5b12fa214620c3a29917097 +Subproject commit 0674958512c705255d4ec4631d3280068ce0a7af ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-base/master] Update PostgreSQL JDBC driver version.
commit 0674958512c705255d4ec4631d3280068ce0a7af Author: Karsten Loesing Date: Tue Sep 17 09:31:47 2019 +0200 Update PostgreSQL JDBC driver version. There's an issue with Java 9+ complaining about an ilegal reflective access [...] to field java.util.TimeZone.defaultTimeZone which is fixed in PostgreSQL JDBC driver version 42.2.0. Updating to 42.2.5 which happens to be in Debian buster under the assumption that is has seen more usage than 42.2.0 (containing the fix) or 42.2.7 (latest at the moment). Found while running integration tests using metrics-test. --- java/ivy.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/ivy.xml b/java/ivy.xml index e5f07aa..2860b74 100644 --- a/java/ivy.xml +++ b/java/ivy.xml @@ -46,7 +46,7 @@ - + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/master] Update to latest metrics-lib.
commit 37190035e688cc0a313bb06e10324c8fc609a0b6 Author: Karsten Loesing Date: Mon Sep 16 16:39:37 2019 +0200 Update to latest metrics-lib. --- src/submods/metrics-lib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/submods/metrics-lib b/src/submods/metrics-lib index 53b16d1..7e396f0 16 --- a/src/submods/metrics-lib +++ b/src/submods/metrics-lib @@ -1 +1 @@ -Subproject commit 53b16d192da4a7fa5c1987a89a60257061602882 +Subproject commit 7e396f01ef6de4cb3cd7e2b7fc977b4af7b97e7f ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-web/master] Fix two issues found using metrics-test.
commit 5d98d4806e01169f226bace3fc095298844d4427
Author: Karsten Loesing
Date: Mon Sep 16 16:29:10 2019 +0200
Fix two issues found using metrics-test.
- Make unit test locale-independent, similar to Onionoo's 860228c.
- Replace deprecated method Class.newInstance().
---
.../java/org/torproject/metrics/stats/hidserv/Aggregator.java | 5 +++--
.../metrics/stats/hidserv/ComputedNetworkFractions.java| 5 +++--
.../org/torproject/metrics/stats/hidserv/DocumentStore.java| 6 --
.../metrics/stats/hidserv/ExtrapolatedHidServStats.java| 10 ++
.../java/org/torproject/metrics/stats/hidserv/Simulate.java| 9 +
5 files changed, 21 insertions(+), 14 deletions(-)
diff --git a/src/main/java/org/torproject/metrics/stats/hidserv/Aggregator.java
b/src/main/java/org/torproject/metrics/stats/hidserv/Aggregator.java
index 36e7967..12a89a3 100644
--- a/src/main/java/org/torproject/metrics/stats/hidserv/Aggregator.java
+++ b/src/main/java/org/torproject/metrics/stats/hidserv/Aggregator.java
@@ -13,6 +13,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
@@ -176,8 +177,8 @@ public class Aggregator {
/* Put together all aggregated values in a single line. */
String date = e.getKey();
int numStats = weightedValues.size();
-sb.append(String.format("%s,%s,%.0f,%.0f,%.0f,%.8f,%d%n", date,
-type, weightedMean, weightedMedian, weightedInterquartileMean,
+sb.append(String.format(Locale.US, "%s,%s,%.0f,%.0f,%.0f,%.8f,%d%n",
+date, type, weightedMean, weightedMedian,
weightedInterquartileMean,
sumFraction, numStats));
}
}
diff --git
a/src/main/java/org/torproject/metrics/stats/hidserv/ComputedNetworkFractions.java
b/src/main/java/org/torproject/metrics/stats/hidserv/ComputedNetworkFractions.java
index d5fd279..835ac6f 100644
---
a/src/main/java/org/torproject/metrics/stats/hidserv/ComputedNetworkFractions.java
+++
b/src/main/java/org/torproject/metrics/stats/hidserv/ComputedNetworkFractions.java
@@ -8,6 +8,7 @@ import org.slf4j.LoggerFactory;
import java.util.Collections;
import java.util.HashMap;
+import java.util.Locale;
import java.util.Map;
/** Computed fraction of hidden-service activity that a single relay is
@@ -116,9 +117,9 @@ public class ComputedNetworkFractions implements Document {
validAfterDate);
String second = validAfterHour
+ (this.fractionRendRelayedCells == 0.0 ? ","
-: String.format(",%f", this.fractionRendRelayedCells))
+: String.format(Locale.US, ",%f", this.fractionRendRelayedCells))
+ (this.fractionDirOnionsSeen == 0.0 ? ","
-: String.format(",%f", this.fractionDirOnionsSeen));
+: String.format(Locale.US, ",%f", this.fractionDirOnionsSeen));
return new String[] { first, second };
}
diff --git
a/src/main/java/org/torproject/metrics/stats/hidserv/DocumentStore.java
b/src/main/java/org/torproject/metrics/stats/hidserv/DocumentStore.java
index f3163e1..9061aa3 100644
--- a/src/main/java/org/torproject/metrics/stats/hidserv/DocumentStore.java
+++ b/src/main/java/org/torproject/metrics/stats/hidserv/DocumentStore.java
@@ -13,6 +13,7 @@ import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.LineNumberReader;
+import java.lang.reflect.InvocationTargetException;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
@@ -136,7 +137,7 @@ public class DocumentStore {
&& !formattedString0.startsWith(prefix)) {
/* Skip line not starting with prefix. */
} else {
- T document = this.clazz.newInstance();
+ T document = this.clazz.getDeclaredConstructor().newInstance();
if (!document.parse(new String[] { formattedString0,
line.substring(1) })) {
log.warn("Unable to read line {} from {}. Not retrieving any "
@@ -151,7 +152,8 @@ public class DocumentStore {
log.warn("Unable to read {}. Not retrieving any previously stored "
+ "documents.", documentFile.getAbsolutePath(), e);
return null;
-} catch (InstantiationException | IllegalAccessException e) {
+} catch (InstantiationException | IllegalAccessException
+| NoSuchMethodException | InvocationTargetException e) {
log.warn("Unable to read {}. Cannot instantiate document object.",
documentFile.getAbsolutePath(), e);
return null;
diff --git
a/src/main/java/org/torproject/metrics/stats/hidserv/ExtrapolatedHidServStats.java
b/src/main/java/org/torproject/metrics/stats/hidserv/ExtrapolatedHidServStats.java
index
[tor-commits] [metrics-web/master] Update CollecTor page to include snowflake stats.
commit 412ac16376bb93e9be80b776a64435940f16681d Author: Karsten Loesing Date: Wed Aug 14 10:58:38 2019 +0200 Update CollecTor page to include snowflake stats. Implements #29461. --- src/main/resources/web/jsps/collector.jsp | 24 1 file changed, 24 insertions(+) diff --git a/src/main/resources/web/jsps/collector.jsp b/src/main/resources/web/jsps/collector.jsp index a05b7e0..42280a9 100644 --- a/src/main/resources/web/jsps/collector.jsp +++ b/src/main/resources/web/jsps/collector.jsp @@ -196,6 +196,15 @@ recent archive + + Snowflake Statistics + + + Snowflake Statistics + @type snowflake-stats 1.0 + recent + archive + @@ -792,6 +801,21 @@ These are available from a DirPort's + +Snowflake Statistics +# + + +Snowflake Statistics +@type snowflake-stats 1.0 + recent + archive +# + + + +Snowflake statistics containing aggregated information about snowflake proxies and snowflake clients as generated by the snowflake broker. + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/master] Bump version to 1.10.0-dev.
commit 16894a4d0096ac76e0ccbba1cd7b593ab1ddab2c Author: Karsten Loesing Date: Thu Sep 12 10:59:05 2019 +0200 Bump version to 1.10.0-dev. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index f4812c4..4bada49 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,7 @@ - + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/release] Archive snowflake statistics.
commit 569bb83ff5e50843079ebd1150801a8bd8efa747
Author: Karsten Loesing
Date: Wed Aug 14 08:51:29 2019 +0200
Archive snowflake statistics.
Implements #29461.
---
CHANGELOG.md | 2 +
build.xml | 2 +-
.../org/torproject/metrics/collector/Main.java | 3 +
.../metrics/collector/conf/Annotation.java | 3 +-
.../metrics/collector/conf/Configuration.java | 1 +
.../org/torproject/metrics/collector/conf/Key.java | 8 +-
.../persist/SnowflakeStatsPersistence.java | 37
.../snowflake/SnowflakeStatsDownloader.java| 191 +
.../metrics/collector/sync/SyncPersistence.java| 5 +
src/main/resources/collector.properties| 20 +++
src/main/resources/create-tarballs.sh | 7 +
src/main/resources/docs/PROTOCOL | 33 +++-
.../metrics/collector/conf/ConfigurationTest.java | 2 +-
.../metrics/collector/cron/CollecTorMainTest.java | 1 +
.../metrics/collector/cron/SchedulerTest.java | 9 +-
15 files changed, 317 insertions(+), 7 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4fd0401..c537d22 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,8 @@
versions resolved by Ivy are the same as in Debian stretch with
few exceptions.
- Remove Cobertura from the build process.
+ - Archive snowflake statistics.
+ - Update to metrics-lib 2.7.0.
# Changes in version 1.9.1 - 2019-05-29
diff --git a/build.xml b/build.xml
index 39180c7..d1d5b70 100644
--- a/build.xml
+++ b/build.xml
@@ -12,7 +12,7 @@
-
+
diff --git a/src/main/java/org/torproject/metrics/collector/Main.java
b/src/main/java/org/torproject/metrics/collector/Main.java
index 46e93af..6907e93 100644
--- a/src/main/java/org/torproject/metrics/collector/Main.java
+++ b/src/main/java/org/torproject/metrics/collector/Main.java
@@ -14,6 +14,7 @@ import
org.torproject.metrics.collector.exitlists.ExitListDownloader;
import org.torproject.metrics.collector.indexer.CreateIndexJson;
import org.torproject.metrics.collector.onionperf.OnionPerfDownloader;
import org.torproject.metrics.collector.relaydescs.ArchiveWriter;
+import org.torproject.metrics.collector.snowflake.SnowflakeStatsDownloader;
import org.torproject.metrics.collector.webstats.SanitizeWeblogs;
import org.slf4j.Logger;
@@ -53,6 +54,8 @@ public class Main {
collecTorMains.put(Key.RelaydescsActivated, ArchiveWriter.class);
collecTorMains.put(Key.OnionPerfActivated, OnionPerfDownloader.class);
collecTorMains.put(Key.WebstatsActivated, SanitizeWeblogs.class);
+collecTorMains.put(Key.SnowflakeStatsActivated,
+SnowflakeStatsDownloader.class);
}
private static Configuration conf = new Configuration();
diff --git
a/src/main/java/org/torproject/metrics/collector/conf/Annotation.java
b/src/main/java/org/torproject/metrics/collector/conf/Annotation.java
index 2e47df0..8cd3324 100644
--- a/src/main/java/org/torproject/metrics/collector/conf/Annotation.java
+++ b/src/main/java/org/torproject/metrics/collector/conf/Annotation.java
@@ -18,7 +18,8 @@ public enum Annotation {
Server("@type server-descriptor 1.0\n"),
Status("@type bridge-network-status 1.2\n"),
OnionPerf("@type torperf 1.1\n"),
- Vote("@type network-status-vote-3 1.0\n");
+ Vote("@type network-status-vote-3 1.0\n"),
+ SnowflakeStats("@type snowflake-stats 1.0\n");
private final String annotation;
private final byte[] bytes;
diff --git
a/src/main/java/org/torproject/metrics/collector/conf/Configuration.java
b/src/main/java/org/torproject/metrics/collector/conf/Configuration.java
index 69d3bcd..27f5125 100644
--- a/src/main/java/org/torproject/metrics/collector/conf/Configuration.java
+++ b/src/main/java/org/torproject/metrics/collector/conf/Configuration.java
@@ -91,6 +91,7 @@ public class Configuration extends Observable implements
Cloneable {
|| this.getBool(Key.ExitlistsActivated)
|| this.getBool(Key.UpdateindexActivated)
|| this.getBool(Key.OnionPerfActivated)
+|| this.getBool(Key.SnowflakeStatsActivated)
|| this.getBool(Key.WebstatsActivated))) {
throw new ConfigurationException("Nothing is activated!\n"
+ "Please edit collector.properties. Exiting.");
diff --git a/src/main/java/org/torproject/metrics/collector/conf/Key.java
b/src/main/java/org/torproject/metrics/collector/conf/Key.java
index ba4bcd9..e683fe2 100644
--- a/src/main/java/org/torproject/metrics/collector/conf/Key.java
+++ b/src/main/java/org/torproject/metrics/collector/conf/Key.java
@@ -66,7 +66,13 @@ public enum Key {
WebstatsActivated(Boolean.class),
WebstatsLimits(Boolean.class),
WebstatsOffsetMinutes(Integer.class),
- WebstatsPeriodMinutes(Integer.class);
+ WebstatsPeriodMinutes(Integer.c
[tor-commits] [collector/release] Upgrade to latest metrics-base.
commit 02b709b65eb8679020e9af67905640a46c362fec
Author: Karsten Loesing
Date: Wed Jul 24 19:08:55 2019 +0200
Upgrade to latest metrics-base.
Also include change log entries and fix a few Checkstyle complaints.
---
CHANGELOG.md | 16
src/build| 2 +-
.../java/org/torproject/metrics/collector/conf/Key.java | 4 +++-
.../metrics/collector/webstats/SanitizeWeblogs.java | 2 +-
4 files changed, 21 insertions(+), 3 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4682fef..55118c2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,19 @@
+# Changes in version 1.10.0 - 2019-0?-??
+
+ * Medium changes
+ * Update build-time and run-time dependencies to versions found in
+ Debian buster, including: commons-codec-1.11.jar,
+ commons-compress-1.18.jar, jackson-annotations-2.9.8.jar,
+ jackson-core-2.9.8.jar, jackson-databind-2.9.8.jar,
+ logback-core-1.2.3.jar, logback-classic-1.2.3.jar,
+ slf4j-api-1.7.25.jar, and xz-1.8.jar.
+ * Update Checkstyle dependencies to versions found in Debian
+ buster, including: antlr-2.7.7.jar, antlr4-runtime-4.7.2.jar,
+ checkstyle-8.15.jar, commons-beanutils-1.9.3.jar,
+ commons-collections3-3.2.2.jar, commons-logging-1.2.jar,
+ guava-19.0.jar, java-atk-wrapper.jar, and picocli-3.9.2.jar.
+
+
# Changes in version 1.9.1 - 2019-05-29
* Medium changes
diff --git a/src/build b/src/build
index e12ba96..42ee1fc 16
--- a/src/build
+++ b/src/build
@@ -1 +1 @@
-Subproject commit e12ba96aa0708b20375cf7cf5d20dfa19940d18a
+Subproject commit 42ee1fc086b0e86cddf56475124a547465484321
diff --git a/src/main/java/org/torproject/metrics/collector/conf/Key.java
b/src/main/java/org/torproject/metrics/collector/conf/Key.java
index ba4bcd9..32e20ce 100644
--- a/src/main/java/org/torproject/metrics/collector/conf/Key.java
+++ b/src/main/java/org/torproject/metrics/collector/conf/Key.java
@@ -72,7 +72,9 @@ public enum Key {
private static Set keys;
/**
- * @param Class of key value.
+ * Instantiate a new {@code Key} using the given class for the key value.
+ *
+ * @param clazz Class of key value.
*/
Key(Class clazz) {
this.clazz = clazz;
diff --git
a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java
b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java
index 027cfde..539194e 100644
---
a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java
+++
b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java
@@ -166,7 +166,7 @@ public class SanitizeWeblogs extends CollecTorMain {
private static final int BATCH = 100_000;
static byte[] toCompressedBytes(Map lines)
-throws DescriptorParseException {
+ throws DescriptorParseException {
try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
OutputStream os = FileType.XZ.outputStream(baos)) {
for (Map.Entry entry : lines.entrySet()) {
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/release] Revert all Debian buster library upgrades.
commit 6ad185caeaafa3c67113cb6ea6290e984feda6ee
Author: Karsten Loesing
Date: Sun Jul 28 21:50:09 2019 +0200
Revert all Debian buster library upgrades.
Turns out that updating all dependencies, including those in
metrics-web and exonerator using servlets and JSPs, is much harder
than expected. We decided to revert all these changes for now, so that
we can merge patches again. We're going to investigate alternatives
like Ant + Ivy, Maven, or Gradle in the near future.
---
CHANGELOG.md | 20
build.xml| 18 +-
src/build| 2 +-
.../org/torproject/metrics/collector/conf/Key.java | 4 +---
.../metrics/collector/webstats/SanitizeWeblogs.java | 2 +-
5 files changed, 12 insertions(+), 34 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3a07711..4682fef 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,23 +1,3 @@
-# Changes in version 1.10.0 - 2019-0?-??
-
- * Medium changes
- * Update build-time and run-time dependencies to versions found in
- Debian buster, including: commons-codec-1.11.jar,
- commons-compress-1.18.jar, jackson-annotations-2.9.8.jar,
- jackson-core-2.9.8.jar, jackson-databind-2.9.8.jar,
- logback-core-1.2.3.jar, logback-classic-1.2.3.jar,
- slf4j-api-1.7.25.jar, and xz-1.8.jar.
- * Update Checkstyle dependencies to versions found in Debian
- buster, including: antlr-2.7.7.jar, antlr4-runtime-4.7.2.jar,
- checkstyle-8.15.jar, commons-beanutils-1.9.3.jar,
- commons-collections3-3.2.2.jar, commons-logging-1.2.jar,
- guava-19.0.jar, java-atk-wrapper.jar, and picocli-3.9.2.jar.
- * Update Cobertura dependencies to versions found in Debian buster,
- including: asm4-5.0.4.jar, asm4-util-5.0.4.jar,
- asm4-tree-5.0.4.jar, asm4-commons-5.0.4.jar,
- asm4-analysis-5.0.4.jar, and commons-lang3-3.8.jar
-
-
# Changes in version 1.9.1 - 2019-05-29
* Medium changes
diff --git a/build.xml b/build.xml
index d35fd58..d0fd248 100644
--- a/build.xml
+++ b/build.xml
@@ -15,16 +15,16 @@
-
-
-
-
-
-
+
+
+
+
+
+
-
-
-
+
+
+
diff --git a/src/build b/src/build
index 42ee1fc..ed48192 16
--- a/src/build
+++ b/src/build
@@ -1 +1 @@
-Subproject commit 42ee1fc086b0e86cddf56475124a547465484321
+Subproject commit ed4819215aa60cc8935f955d9404f30e83dd892b
diff --git a/src/main/java/org/torproject/metrics/collector/conf/Key.java
b/src/main/java/org/torproject/metrics/collector/conf/Key.java
index 32e20ce..ba4bcd9 100644
--- a/src/main/java/org/torproject/metrics/collector/conf/Key.java
+++ b/src/main/java/org/torproject/metrics/collector/conf/Key.java
@@ -72,9 +72,7 @@ public enum Key {
private static Set keys;
/**
- * Instantiate a new {@code Key} using the given class for the key value.
- *
- * @param clazz Class of key value.
+ * @param Class of key value.
*/
Key(Class clazz) {
this.clazz = clazz;
diff --git
a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java
b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java
index 539194e..027cfde 100644
---
a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java
+++
b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java
@@ -166,7 +166,7 @@ public class SanitizeWeblogs extends CollecTorMain {
private static final int BATCH = 100_000;
static byte[] toCompressedBytes(Map lines)
- throws DescriptorParseException {
+throws DescriptorParseException {
try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
OutputStream os = FileType.XZ.outputStream(baos)) {
for (Map.Entry entry : lines.entrySet()) {
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/release] Prepare for 1.10.0 release.
commit 1037677d9ec1c94a26d8dccff454e348d29bd344 Author: Karsten Loesing Date: Thu Sep 12 10:50:31 2019 +0200 Prepare for 1.10.0 release. --- CHANGELOG.md | 2 +- build.xml| 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c537d22..b38b124 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changes in version 1.10.0 - 2019-??-?? +# Changes in version 1.10.0 - 2019-09-12 * Medium changes - Changed local import of bandwidth files to include the parent diff --git a/build.xml b/build.xml index d1d5b70..f4812c4 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,7 @@ - + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/release] Also update metrics-lib version for GitLab's CI.
commit d132d40f8b3723e41ce7baf32d624a38d08edeba Author: Karsten Loesing Date: Thu Sep 12 10:20:12 2019 +0200 Also update metrics-lib version for GitLab's CI. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b67b5a4..5bd5623 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,6 @@ variables: GIT_STRATEGY: clone - METRICS_LIB_VERSION: "2.6.2" + METRICS_LIB_VERSION: "2.7.0" JAVA_TOOL_OPTIONS: "-Dfile.encoding=UTF-8" stages: ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/release] Add source name to locally imported bandwidth files.
commit abde3f7591117060ccf1afc9b26531d79619301f
Author: Karsten Loesing
Date: Wed Jul 10 10:34:09 2019 +0200
Add source name to locally imported bandwidth files.
Implements #30219.
---
CHANGELOG.md | 7 +++
.../metrics/collector/relaydescs/ArchiveReader.java | 2 +-
.../metrics/collector/relaydescs/ArchiveWriter.java | 12 +---
.../collector/relaydescs/CachedRelayDescriptorReader.java| 6 +++---
.../collector/relaydescs/RelayDescriptorDownloader.java | 4 ++--
.../metrics/collector/relaydescs/RelayDescriptorParser.java | 7 +--
src/main/resources/collector.properties | 5 -
7 files changed, 31 insertions(+), 12 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4682fef..613ef71 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,10 @@
+# Changes in version 1.10.0 - 2019-??-??
+
+ * Medium changes
+ - Changed local import of bandwidth files to include the parent
+ directory name as @source annotation and to the filename.
+
+
# Changes in version 1.9.1 - 2019-05-29
* Medium changes
diff --git
a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java
b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java
index 27010a9..fee46c9 100644
---
a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java
+++
b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java
@@ -141,7 +141,7 @@ public class ArchiveReader {
}
bis.close();
byte[] allData = baos.toByteArray();
-boolean stored = this.rdp.parse(allData);
+boolean stored = this.rdp.parse(allData, pop);
if (!stored) {
filesToRetry.add(pop);
continue;
diff --git
a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
index e1279ee..911ac2a 100644
---
a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
+++
b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
@@ -745,18 +745,24 @@ public class ArchiveWriter extends CollecTorMain {
/** Stores a bandwidth file to disk. */
void storeBandwidthFile(byte[] data, LocalDateTime fileCreatedOrTimestamp,
- String bandwidthFileDigest) {
+ String sourceName, String bandwidthFileDigest) {
DateTimeFormatter printFormat = DateTimeFormatter
.ofPattern("/MM/dd/-MM-dd-HH-mm-ss").withZone(ZoneOffset.UTC);
File tarballFile = Paths.get(this.outputDirectory, "bandwidth",
fileCreatedOrTimestamp.format(printFormat) + "-bandwidth-"
++ (null == sourceName ? "" : (sourceName + "-"))
+ bandwidthFileDigest).toFile();
+StringBuilder sb = new StringBuilder();
+sb.append(Annotation.BandwidthFile.toString());
+if (null != sourceName) {
+ sb.append("@source ").append(sourceName).append('\n');
+}
boolean tarballFileExistedBefore = tarballFile.exists();
File rsyncFile = Paths.get(recentPathName, RELAY_DESCRIPTORS, "bandwidths",
tarballFile.getName()).toFile();
File[] outputFiles = new File[] { tarballFile, rsyncFile };
-if (this.store(Annotation.BandwidthFile.bytes(), data, outputFiles, null))
{
- this.storedVotesCounter++;
+if (this.store(sb.toString().getBytes(), data, outputFiles, null)) {
+ this.storedBandwidthsCounter++;
}
if (!tarballFileExistedBefore
&& this.nowLocalDateTime.isAfter(fileCreatedOrTimestamp.plusDays(3L)))
{
diff --git
a/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java
b/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java
index 3e1bba4..0308f05 100644
---
a/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java
+++
b/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java
@@ -158,7 +158,7 @@ public class CachedRelayDescriptorReader {
allData));
if (!this.lastImportHistory.contains(digest)
&& !this.currentImportHistory.contains(digest)) {
- this.rdp.parse(allData);
+ this.rdp.parse(allData, null);
} else {
this.dumpStats.append(" (skipped)");
}
@@ -183,7 +183,7 @@ public class CachedRelayDescriptorReader {
rawNetworkStatusBytes));
if (!this.lastImportHistory.contains(digest)
&& !this.currentImportHistory.contains(digest)) {
- this.rdp.parse(rawNetworkStatusBytes);
+ this.rdp.parse(rawNetworkStatusBytes, null);
[tor-commits] [collector/release] Use Ivy for resolving external dependencies.
commit 027a54bfe2c970a72db2afc485d2fa12f77cb247 Author: Karsten Loesing Date: Tue Aug 6 17:04:41 2019 +0200 Use Ivy for resolving external dependencies. Includes an upgrade to the latest metrics-base commit 256e619. Implements part of #31326. --- CHANGELOG.md | 9 + build.xml| 3 ++- src/build| 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 613ef71..66e53ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,15 @@ * Medium changes - Changed local import of bandwidth files to include the parent directory name as @source annotation and to the filename. + - Use Ivy for resolving external dependencies rather than relying + on files found in Debian stable packages. Requires installing Ivy + (using `apt-get install ivy`, `brew install ivy`, or similar) and + running `ant resolve` (or `ant -lib /usr/share/java resolve`). + Retrieved files are then copied to the `lib/` directory, except + for dependencies on other metrics libraries that still need to be + copied to the `lib/` directory manually. Current dependency + versions resolved by Ivy are the same as in Debian stretch with + few exceptions. # Changes in version 1.9.1 - 2019-05-29 diff --git a/build.xml b/build.xml index d0fd248..39180c7 100644 --- a/build.xml +++ b/build.xml @@ -4,7 +4,8 @@ ]> - + diff --git a/src/build b/src/build index ed48192..256e619 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit ed4819215aa60cc8935f955d9404f30e83dd892b +Subproject commit 256e6192ad0066e48c6abb0ee9ac71714c7a3a0f ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/release] Adds GitLab CI for tests and checks
commit a8a4961a1b49a3104dd37ac93cc41e3f908e25c5 Author: Iain R. Learmonth Date: Thu Sep 5 12:21:43 2019 +0100 Adds GitLab CI for tests and checks Configuration is held in the .gitlab-ci.yml file. Fixes: #31399 --- .gitlab-ci.yml | 25 + 1 file changed, 25 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000..b67b5a4 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,25 @@ +variables: + GIT_STRATEGY: clone + METRICS_LIB_VERSION: "2.6.2" + JAVA_TOOL_OPTIONS: "-Dfile.encoding=UTF-8" + +stages: + - test + +test: + stage: test + image: debian:buster + script: + - apt update + - apt -y install default-jdk ant ivy git curl + - git submodule init + - git submodule update + - mkdir lib + - mkdir tmp + - pushd tmp + - curl https://dist.torproject.org/metrics-lib/$METRICS_LIB_VERSION/metrics-lib-$METRICS_LIB_VERSION.tar.gz | tar xzf - + - popd + - mv tmp/metrics-lib-$METRICS_LIB_VERSION/generated/dist/metrics-lib-$METRICS_LIB_VERSION-thin.jar lib + - ant -lib /usr/share/java resolve + - ant test + - ant checks ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/release] Remove Cobertura from the build process.
commit 801008431115951fe3b0f967c7d4ca0469610ae8 Author: Karsten Loesing Date: Wed Aug 14 11:38:40 2019 +0200 Remove Cobertura from the build process. --- CHANGELOG.md | 1 + src/build| 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66e53ed..4fd0401 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ copied to the `lib/` directory manually. Current dependency versions resolved by Ivy are the same as in Debian stretch with few exceptions. + - Remove Cobertura from the build process. # Changes in version 1.9.1 - 2019-05-29 diff --git a/src/build b/src/build index 256e619..75cda35 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 256e6192ad0066e48c6abb0ee9ac71714c7a3a0f +Subproject commit 75cda35ba7375ab0f5b12fa214620c3a29917097 ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/release] Update to Debian buster libraries.
commit e65269dfe70312e50472de807dc0714b0aed5ec1 Author: Karsten Loesing Date: Wed Jul 24 10:43:52 2019 +0200 Update to Debian buster libraries. Also upgrade to latest metrics-base. Checkstyle is excluded from this update. --- build.xml | 18 +- src/build | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/build.xml b/build.xml index d0fd248..d35fd58 100644 --- a/build.xml +++ b/build.xml @@ -15,16 +15,16 @@ - - - - - - + + + + + + - - - + + + diff --git a/src/build b/src/build index e639c69..e12ba96 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit e639c697e9e94c6dbb26e946e5247c20a62c0661 +Subproject commit e12ba96aa0708b20375cf7cf5d20dfa19940d18a ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/release] Bump version to 1.9.1-dev.
commit d992d8719d507fa8fffee24faa18546da848e16e Author: Karsten Loesing Date: Thu May 30 15:32:07 2019 +0200 Bump version to 1.9.1-dev. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index d2d1f73..d0fd248 100644 --- a/build.xml +++ b/build.xml @@ -8,7 +8,7 @@ - + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/release] Add change log entry for Cobertura deps.
commit 7f36a2b12eb5e25c441e0d8a40c5c9a7de19821c Author: Karsten Loesing Date: Wed Jul 24 19:56:18 2019 +0200 Add change log entry for Cobertura deps. --- CHANGELOG.md | 4 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55118c2..3a07711 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ checkstyle-8.15.jar, commons-beanutils-1.9.3.jar, commons-collections3-3.2.2.jar, commons-logging-1.2.jar, guava-19.0.jar, java-atk-wrapper.jar, and picocli-3.9.2.jar. + * Update Cobertura dependencies to versions found in Debian buster, + including: asm4-5.0.4.jar, asm4-util-5.0.4.jar, + asm4-tree-5.0.4.jar, asm4-commons-5.0.4.jar, + asm4-analysis-5.0.4.jar, and commons-lang3-3.8.jar # Changes in version 1.9.1 - 2019-05-29 ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/master] Prepare for 1.10.0 release.
commit 1037677d9ec1c94a26d8dccff454e348d29bd344 Author: Karsten Loesing Date: Thu Sep 12 10:50:31 2019 +0200 Prepare for 1.10.0 release. --- CHANGELOG.md | 2 +- build.xml| 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c537d22..b38b124 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changes in version 1.10.0 - 2019-??-?? +# Changes in version 1.10.0 - 2019-09-12 * Medium changes - Changed local import of bandwidth files to include the parent diff --git a/build.xml b/build.xml index d1d5b70..f4812c4 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,7 @@ - + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/master] Also update metrics-lib version for GitLab's CI.
commit d132d40f8b3723e41ce7baf32d624a38d08edeba Author: Karsten Loesing Date: Thu Sep 12 10:20:12 2019 +0200 Also update metrics-lib version for GitLab's CI. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b67b5a4..5bd5623 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,6 @@ variables: GIT_STRATEGY: clone - METRICS_LIB_VERSION: "2.6.2" + METRICS_LIB_VERSION: "2.7.0" JAVA_TOOL_OPTIONS: "-Dfile.encoding=UTF-8" stages: ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [collector/master] Archive snowflake statistics.
commit 569bb83ff5e50843079ebd1150801a8bd8efa747
Author: Karsten Loesing
Date: Wed Aug 14 08:51:29 2019 +0200
Archive snowflake statistics.
Implements #29461.
---
CHANGELOG.md | 2 +
build.xml | 2 +-
.../org/torproject/metrics/collector/Main.java | 3 +
.../metrics/collector/conf/Annotation.java | 3 +-
.../metrics/collector/conf/Configuration.java | 1 +
.../org/torproject/metrics/collector/conf/Key.java | 8 +-
.../persist/SnowflakeStatsPersistence.java | 37
.../snowflake/SnowflakeStatsDownloader.java| 191 +
.../metrics/collector/sync/SyncPersistence.java| 5 +
src/main/resources/collector.properties| 20 +++
src/main/resources/create-tarballs.sh | 7 +
src/main/resources/docs/PROTOCOL | 33 +++-
.../metrics/collector/conf/ConfigurationTest.java | 2 +-
.../metrics/collector/cron/CollecTorMainTest.java | 1 +
.../metrics/collector/cron/SchedulerTest.java | 9 +-
15 files changed, 317 insertions(+), 7 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4fd0401..c537d22 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,8 @@
versions resolved by Ivy are the same as in Debian stretch with
few exceptions.
- Remove Cobertura from the build process.
+ - Archive snowflake statistics.
+ - Update to metrics-lib 2.7.0.
# Changes in version 1.9.1 - 2019-05-29
diff --git a/build.xml b/build.xml
index 39180c7..d1d5b70 100644
--- a/build.xml
+++ b/build.xml
@@ -12,7 +12,7 @@
-
+
diff --git a/src/main/java/org/torproject/metrics/collector/Main.java
b/src/main/java/org/torproject/metrics/collector/Main.java
index 46e93af..6907e93 100644
--- a/src/main/java/org/torproject/metrics/collector/Main.java
+++ b/src/main/java/org/torproject/metrics/collector/Main.java
@@ -14,6 +14,7 @@ import
org.torproject.metrics.collector.exitlists.ExitListDownloader;
import org.torproject.metrics.collector.indexer.CreateIndexJson;
import org.torproject.metrics.collector.onionperf.OnionPerfDownloader;
import org.torproject.metrics.collector.relaydescs.ArchiveWriter;
+import org.torproject.metrics.collector.snowflake.SnowflakeStatsDownloader;
import org.torproject.metrics.collector.webstats.SanitizeWeblogs;
import org.slf4j.Logger;
@@ -53,6 +54,8 @@ public class Main {
collecTorMains.put(Key.RelaydescsActivated, ArchiveWriter.class);
collecTorMains.put(Key.OnionPerfActivated, OnionPerfDownloader.class);
collecTorMains.put(Key.WebstatsActivated, SanitizeWeblogs.class);
+collecTorMains.put(Key.SnowflakeStatsActivated,
+SnowflakeStatsDownloader.class);
}
private static Configuration conf = new Configuration();
diff --git
a/src/main/java/org/torproject/metrics/collector/conf/Annotation.java
b/src/main/java/org/torproject/metrics/collector/conf/Annotation.java
index 2e47df0..8cd3324 100644
--- a/src/main/java/org/torproject/metrics/collector/conf/Annotation.java
+++ b/src/main/java/org/torproject/metrics/collector/conf/Annotation.java
@@ -18,7 +18,8 @@ public enum Annotation {
Server("@type server-descriptor 1.0\n"),
Status("@type bridge-network-status 1.2\n"),
OnionPerf("@type torperf 1.1\n"),
- Vote("@type network-status-vote-3 1.0\n");
+ Vote("@type network-status-vote-3 1.0\n"),
+ SnowflakeStats("@type snowflake-stats 1.0\n");
private final String annotation;
private final byte[] bytes;
diff --git
a/src/main/java/org/torproject/metrics/collector/conf/Configuration.java
b/src/main/java/org/torproject/metrics/collector/conf/Configuration.java
index 69d3bcd..27f5125 100644
--- a/src/main/java/org/torproject/metrics/collector/conf/Configuration.java
+++ b/src/main/java/org/torproject/metrics/collector/conf/Configuration.java
@@ -91,6 +91,7 @@ public class Configuration extends Observable implements
Cloneable {
|| this.getBool(Key.ExitlistsActivated)
|| this.getBool(Key.UpdateindexActivated)
|| this.getBool(Key.OnionPerfActivated)
+|| this.getBool(Key.SnowflakeStatsActivated)
|| this.getBool(Key.WebstatsActivated))) {
throw new ConfigurationException("Nothing is activated!\n"
+ "Please edit collector.properties. Exiting.");
diff --git a/src/main/java/org/torproject/metrics/collector/conf/Key.java
b/src/main/java/org/torproject/metrics/collector/conf/Key.java
index ba4bcd9..e683fe2 100644
--- a/src/main/java/org/torproject/metrics/collector/conf/Key.java
+++ b/src/main/java/org/torproject/metrics/collector/conf/Key.java
@@ -66,7 +66,13 @@ public enum Key {
WebstatsActivated(Boolean.class),
WebstatsLimits(Boolean.class),
WebstatsOffsetMinutes(Integer.class),
- WebstatsPeriodMinutes(Integer.class);
+ WebstatsPeriodMinutes(Integer.c
[tor-commits] [metrics-lib/master] Bump version to 2.7.0-dev.
commit 7e396f01ef6de4cb3cd7e2b7fc977b4af7b97e7f Author: Karsten Loesing Date: Wed Sep 11 16:08:16 2019 +0200 Bump version to 2.7.0-dev. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index 04f79fd..02ccf52 100644 --- a/build.xml +++ b/build.xml @@ -7,7 +7,7 @@ - + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-lib/release] Add new SnowflakeStats descriptor type.
commit dc946fe1c8b50386f60f29a5073046eb2c0b27ad
Author: Karsten Loesing
Date: Tue Aug 13 15:01:45 2019 +0200
Add new SnowflakeStats descriptor type.
Implements #29461.
---
CHANGELOG.md | 1 +
.../org/torproject/descriptor/SnowflakeStats.java | 86 ++
.../descriptor/impl/DescriptorParserImpl.java | 5 +
.../java/org/torproject/descriptor/impl/Key.java | 6 +
.../torproject/descriptor/impl/ParseHelper.java| 21 +++
.../descriptor/impl/SnowflakeStatsImpl.java| 185 +
.../org/torproject/descriptor/DescriptorTest.java | 7 +-
.../descriptor/impl/SnowflakeStatsImplTest.java| 137 +++
src/test/resources/snowflake/example_metrics.log | 12 ++
9 files changed, 459 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b804ceb..2fdc45b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,7 @@
versions resolved by Ivy are the same as in Debian stretch with
few exceptions.
- Remove Cobertura from the build process.
+ - Add new SnowflakeStats descriptor type.
# Changes in version 2.6.2 - 2019-05-29
diff --git a/src/main/java/org/torproject/descriptor/SnowflakeStats.java
b/src/main/java/org/torproject/descriptor/SnowflakeStats.java
new file mode 100644
index 000..379a0f7
--- /dev/null
+++ b/src/main/java/org/torproject/descriptor/SnowflakeStats.java
@@ -0,0 +1,86 @@
+/* Copyright 2019 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.descriptor;
+
+import java.time.Duration;
+import java.time.LocalDateTime;
+import java.util.Optional;
+import java.util.SortedMap;
+
+/**
+ * Contain aggregated information about snowflake proxies and snowflake
clients.
+ *
+ * @since 2.7.0
+ */
+public interface SnowflakeStats extends Descriptor {
+
+ /**
+ * Return the end of the included measurement interval.
+ *
+ * @return End of the included measurement interval.
+ * @since 2.7.0
+ */
+ LocalDateTime snowflakeStatsEnd();
+
+ /**
+ * Return the length of the included measurement interval.
+ *
+ * @return Length of the included measurement interval.
+ * @since 2.7.0
+ */
+ Duration snowflakeStatsIntervalLength();
+
+ /**
+ * Return a list of mappings from two-letter country codes to the number of
+ * unique IP addresses of snowflake proxies that have polled.
+ *
+ * @return List of mappings from two-letter country codes to the number of
+ * unique IP addresses of snowflake proxies that have polled.
+ * @since 2.7.0
+ */
+ Optional> snowflakeIps();
+
+ /**
+ * Return a count of the total number of unique IP addresses of snowflake
+ * proxies that have polled.
+ *
+ * @return Count of the total number of unique IP addresses of snowflake
+ * proxies that have polled.
+ * @since 2.7.0
+ */
+ Optional snowflakeIpsTotal();
+
+ /**
+ * Return a count of the number of times a proxy has polled but received no
+ * client offer, rounded up to the nearest multiple of 8.
+ *
+ * @return Count of the number of times a proxy has polled but received no
+ * client offer, rounded up to the nearest multiple of 8.
+ * @since 2.7.0
+ */
+ Optional snowflakeIdleCount();
+
+ /**
+ * Return a count of the number of times a client has requested a proxy from
+ * the broker but no proxies were available, rounded up to the nearest
+ * multiple of 8.
+ *
+ * @return Count of the number of times a client has requested a proxy from
+ * the broker but no proxies were available, rounded up to the nearest
+ * multiple of 8.
+ * @since 2.7.0
+ */
+ Optional clientDeniedCount();
+
+ /**
+ * Return a count of the number of times a client successfully received a
+ * proxy from the broker, rounded up to the nearest multiple of 8.
+ *
+ * @return Count of the number of times a client successfully received a
proxy
+ * from the broker, rounded up to the nearest multiple of 8.
+ * @since 2.7.0
+ */
+ Optional clientSnowflakeMatchCount();
+}
+
diff --git
a/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java
b/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java
index ef9da56..9b620cb 100644
--- a/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java
@@ -130,6 +130,11 @@ public class DescriptorParserImpl implements
DescriptorParser {
} else if (firstLines.startsWith("@type torperf 1.")) {
return TorperfResultImpl.parseTorperfResults(rawDescriptorBytes,
sourceFile);
+} else if (firstLines.startsWith("@type snowflake-stats 1.")
+|| firstLines.startsWith(Key.SNOWFLAKE_STATS_END.keyword + SP)
+|| firstLines.contains(NL + Key.SNOWFLAKE_STATS_END.keyword + SP)) {
+ return this.parseOneOrMoreDescriptors(rawDesc
[tor-commits] [metrics-lib/release] Prepare for 2.7.0 release.
commit 0520779255dc44d30a69a25f24f5a638ae870a6e Author: Karsten Loesing Date: Fri Sep 6 08:53:14 2019 +0200 Prepare for 2.7.0 release. --- CHANGELOG.md | 2 +- build.xml| 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fdc45b..457533e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changes in version 2.7.0 - 2019-0?-?? +# Changes in version 2.7.0 - 2019-09-06 * Medium changes - Use Ivy for resolving external dependencies rather than relying diff --git a/build.xml b/build.xml index f319cf0..04f79fd 100644 --- a/build.xml +++ b/build.xml @@ -7,7 +7,7 @@ - + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-lib/release] Upgrade to latest metrics-base.
commit aa686c4b492badec430abdc0b674c283e0ed6d39 Author: Karsten Loesing Date: Wed Jul 24 19:25:19 2019 +0200 Upgrade to latest metrics-base. Also include change log entries. --- CHANGELOG.md | 16 src/build| 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18d250e..66a626a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,19 @@ +# Changes in version 2.7.0 - 2019-0?-?? + + * Medium changes + * Update build-time and run-time dependencies to versions found in + Debian buster, including: commons-codec-1.11.jar, + commons-compress-1.18.jar, jackson-annotations-2.9.8.jar, + jackson-core-2.9.8.jar, jackson-databind-2.9.8.jar, + logback-core-1.2.3.jar, logback-classic-1.2.3.jar, + slf4j-api-1.7.25.jar, and xz-1.8.jar. + * Update Checkstyle dependencies to versions found in Debian + buster, including: antlr-2.7.7.jar, antlr4-runtime-4.7.2.jar, + checkstyle-8.15.jar, commons-beanutils-1.9.3.jar, + commons-collections3-3.2.2.jar, commons-logging-1.2.jar, + guava-19.0.jar, java-atk-wrapper.jar, and picocli-3.9.2.jar. + + # Changes in version 2.6.2 - 2019-05-29 * Medium changes diff --git a/src/build b/src/build index e12ba96..42ee1fc 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit e12ba96aa0708b20375cf7cf5d20dfa19940d18a +Subproject commit 42ee1fc086b0e86cddf56475124a547465484321 ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-lib/release] Fix leaking resource in TorperfResultImpl.java
commit 114f548674d8586453577f2a16ad6b39b1714118
Author: fava
Date: Thu Aug 29 06:52:16 2019 +
Fix leaking resource in TorperfResultImpl.java
Fix leaking resource in TorperfResultImpl.java using
try-with-resource statement.
This fix is related to analysis on metrics-lib using sonarqube
Implements part of #30544
Signed-off-by: fava
---
.../descriptor/impl/TorperfResultImpl.java | 45 --
1 file changed, 24 insertions(+), 21 deletions(-)
diff --git
a/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java
b/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java
index 4eb00ac..83e9491 100644
--- a/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java
@@ -30,28 +30,31 @@ public class TorperfResultImpl extends DescriptorImpl
List parsedDescriptors = new ArrayList<>();
String descriptorString = new String(rawDescriptorBytes,
StandardCharsets.UTF_8);
-Scanner scanner = new Scanner(descriptorString).useDelimiter("\r?\n");
-String typeAnnotation = "";
-while (scanner.hasNext()) {
- String line = scanner.next();
- if (line.startsWith("@type torperf ")) {
-String[] parts = line.split(" ");
-if (parts.length != 3) {
- throw new DescriptorParseException("Illegal line '" + line
- + "'.");
-}
-String version = parts[2];
-if (!version.startsWith("1.")) {
- throw new DescriptorParseException("Unsupported version in "
- + " line '" + line + "'.");
+try (Scanner scanner =
+ new Scanner(descriptorString).useDelimiter("\r?\n")) {
+ String typeAnnotation = "";
+ while (scanner.hasNext()) {
+String line = scanner.next();
+if (line.startsWith("@type torperf ")) {
+ String[] parts = line.split(" ");
+ if (parts.length != 3) {
+throw new DescriptorParseException("Illegal line '" + line
++ "'.");
+ }
+ String version = parts[2];
+ if (!version.startsWith("1.")) {
+throw new DescriptorParseException("Unsupported version in "
++ " line '" + line + "'.");
+ }
+ typeAnnotation = line + "\n";
+} else {
+ /* XXX21932 */
+ parsedDescriptors.add(new TorperfResultImpl(
+ (typeAnnotation + line + "\n")
+ .getBytes(StandardCharsets.UTF_8),
+ descriptorFile));
+ typeAnnotation = "";
}
-typeAnnotation = line + "\n";
- } else {
-/* XXX21932 */
-parsedDescriptors.add(new TorperfResultImpl(
-(typeAnnotation + line + "\n").getBytes(StandardCharsets.UTF_8),
-descriptorFile));
-typeAnnotation = "";
}
}
return parsedDescriptors;
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-lib/release] Add change log entry for Cobertura deps.
commit 4ffa637c56d676dc2c0ceeec48556110aee2050a Author: Karsten Loesing Date: Wed Jul 24 19:56:39 2019 +0200 Add change log entry for Cobertura deps. --- CHANGELOG.md | 4 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66a626a..e3622e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ checkstyle-8.15.jar, commons-beanutils-1.9.3.jar, commons-collections3-3.2.2.jar, commons-logging-1.2.jar, guava-19.0.jar, java-atk-wrapper.jar, and picocli-3.9.2.jar. + * Update Cobertura dependencies to versions found in Debian buster, + including: asm4-5.0.4.jar, asm4-util-5.0.4.jar, + asm4-tree-5.0.4.jar, asm4-commons-5.0.4.jar, + asm4-analysis-5.0.4.jar, and commons-lang3-3.8.jar # Changes in version 2.6.2 - 2019-05-29 ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-lib/release] Fix leaking resource in DescriptorReaderImpl.java
commit ce19c16543b410cedf3216ba477537d28cc3c818
Author: fava
Date: Thu Aug 22 05:28:44 2019 +
Fix leaking resource in DescriptorReaderImpl.java
Fix leaking resource in DescriptorReaderImpl.java using
try-with-resource statement.
This fix is related to analysis on metrics-lib using sonarqube
Implements part of #30544
---
.../descriptor/impl/DescriptorReaderImpl.java | 70 --
1 file changed, 37 insertions(+), 33 deletions(-)
diff --git
a/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java
b/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java
index b8eb191..90ef2c6 100644
--- a/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java
@@ -281,41 +281,45 @@ public class DescriptorReaderImpl implements
DescriptorReader {
}
private void readTarball(File file) throws IOException {
- FileInputStream in = new FileInputStream(file);
- if (in.available() <= 0) {
-return;
- }
- TarArchiveInputStream tais;
- if (file.getName().endsWith(".tar.bz2")) {
-tais = new TarArchiveInputStream(new BZip2CompressorInputStream(in));
- } else if (file.getName().endsWith(".tar.xz")) {
-tais = new TarArchiveInputStream(new XZCompressorInputStream(in));
- } else if (file.getName().endsWith(".tar")) {
-tais = new TarArchiveInputStream(in);
- } else {
-return;
- }
- BufferedInputStream bis = new BufferedInputStream(tais);
- TarArchiveEntry tae;
- while ((tae = tais.getNextTarEntry()) != null) {
-if (tae.isDirectory()) {
- continue;
+ try (FileInputStream in = new FileInputStream(file)) {
+if (in.available() <= 0) {
+ return;
}
-ByteArrayOutputStream baos = new ByteArrayOutputStream();
-int len;
-byte[] data = new byte[1024];
-while ((len = bis.read(data, 0, 1024)) >= 0) {
- baos.write(data, 0, len);
+TarArchiveInputStream tais;
+if (file.getName().endsWith(".tar.bz2")) {
+ tais = new TarArchiveInputStream(new BZip2CompressorInputStream(in));
+} else if (file.getName().endsWith(".tar.xz")) {
+ tais = new TarArchiveInputStream(new XZCompressorInputStream(in));
+} else if (file.getName().endsWith(".tar")) {
+ tais = new TarArchiveInputStream(in);
+} else {
+ return;
}
-byte[] rawDescriptorBytes = baos.toByteArray();
-if (rawDescriptorBytes.length < 1) {
- continue;
-}
-String fileName = tae.getName().substring(
-tae.getName().lastIndexOf("/") + 1);
-for (Descriptor descriptor : this.descriptorParser.parseDescriptors(
-rawDescriptorBytes, file, fileName)) {
- this.descriptorQueue.add(descriptor);
+try (BufferedInputStream bis = new BufferedInputStream(tais)) {
+ TarArchiveEntry tae;
+ while ((tae = tais.getNextTarEntry()) != null) {
+if (tae.isDirectory()) {
+ continue;
+}
+try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
+ int len;
+ byte[] data = new byte[1024];
+ while ((len = bis.read(data, 0, 1024)) >= 0) {
+baos.write(data, 0, len);
+ }
+ byte[] rawDescriptorBytes = baos.toByteArray();
+ if (rawDescriptorBytes.length < 1) {
+continue;
+ }
+ String fileName = tae.getName().substring(
+ tae.getName().lastIndexOf("/") + 1);
+ for (Descriptor descriptor :
+ this.descriptorParser.parseDescriptors(
+ rawDescriptorBytes, file, fileName)) {
+this.descriptorQueue.add(descriptor);
+ }
+}
+ }
}
}
}
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-lib/release] Fix leaking resource in ExitListEntryImpl.java
commit 9fc82fafb32dcc39ebf6780822ecbc06d5f952e5
Author: fava
Date: Thu Aug 22 05:41:03 2019 +
Fix leaking resource in ExitListEntryImpl.java
Fix leaking resource in ExitListEntryImpl.java using
try-with-resource statement.
This fix is related to analysis on metrics-lib using sonarqube
Implements part of #30544
---
.../descriptor/impl/ExitListEntryImpl.java | 51 +++---
1 file changed, 26 insertions(+), 25 deletions(-)
diff --git
a/src/main/java/org/torproject/descriptor/impl/ExitListEntryImpl.java
b/src/main/java/org/torproject/descriptor/impl/ExitListEntryImpl.java
index 5389ce3..ba203e2 100644
--- a/src/main/java/org/torproject/descriptor/impl/ExitListEntryImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/ExitListEntryImpl.java
@@ -62,31 +62,32 @@ public class ExitListEntryImpl implements ExitList.Entry {
}
private void parseExitListEntry()
- throws DescriptorParseException {
-Scanner scanner = new Scanner(this.exitListEntryString)
-.useDelimiter(ExitList.EOL);
-while (scanner.hasNext()) {
- String line = scanner.next();
- String[] parts = line.split(" ");
- String keyword = parts[0];
- switch (keyword) {
-case "ExitNode":
- this.parseExitNodeLine(line, parts);
- break;
-case "Published":
- this.parsePublishedLine(line, parts);
- break;
-case "LastStatus":
- this.parseLastStatusLine(line, parts);
- break;
-case "ExitAddress":
- this.parseExitAddressLine(line, parts);
- break;
-default:
- if (this.unrecognizedLines == null) {
-this.unrecognizedLines = new ArrayList<>();
- }
- this.unrecognizedLines.add(line);
+ throws DescriptorParseException {
+try (Scanner scanner = new Scanner(this.exitListEntryString)
+.useDelimiter(ExitList.EOL)) {
+ while (scanner.hasNext()) {
+String line = scanner.next();
+String[] parts = line.split(" ");
+String keyword = parts[0];
+switch (keyword) {
+ case "ExitNode":
+this.parseExitNodeLine(line, parts);
+break;
+ case "Published":
+this.parsePublishedLine(line, parts);
+break;
+ case "LastStatus":
+this.parseLastStatusLine(line, parts);
+break;
+ case "ExitAddress":
+this.parseExitAddressLine(line, parts);
+break;
+ default:
+if (this.unrecognizedLines == null) {
+ this.unrecognizedLines = new ArrayList<>();
+}
+this.unrecognizedLines.add(line);
+}
}
}
}
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-lib/release] Ignore Intellij's project files.
commit 3987fe8fd6164fcc7c35b21d5737e7fe9cd45fc0 Author: fava Date: Thu Aug 22 04:39:20 2019 + Ignore Intellij's project files. Ignore Intellij's project files in order to avoid personal configurations erroneusly committed to the repository. --- .gitignore | 4 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index af0ea90..a941692 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,7 @@ metrics-lib-*.tar.gz .project build.properties *~ + +# IntelliJ files +.idea/ +*.iml ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-lib/release] Update to Debian buster libraries.
commit 453a6f5acb9b9935a88de8c6e39e05e29b5423e7
Author: Karsten Loesing
Date: Thu Jul 18 07:53:39 2019 +0200
Update to Debian buster libraries.
Also upgrade to latest metrics-base.
Note that Checkstyle is excluded from this update, because there's a
yet unresolved issue with the new version: ("Unable to create Root
Module") that we'll have to address in a separate patch. But given
that Checkstyle is not required for the build it's okay to keep the
old version for now. It'll be in the release tarball.
---
build.xml | 14 +++---
src/build | 2 +-
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/build.xml b/build.xml
index 3066e90..a163dc9 100644
--- a/build.xml
+++ b/build.xml
@@ -14,13 +14,13 @@
-
-
-
-
-
-
-
+
+
+
+
+
+
+
diff --git a/src/build b/src/build
index e639c69..e12ba96 16
--- a/src/build
+++ b/src/build
@@ -1 +1 @@
-Subproject commit e639c697e9e94c6dbb26e946e5247c20a62c0661
+Subproject commit e12ba96aa0708b20375cf7cf5d20dfa19940d18a
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-lib/release] Adds GitLab CI for tests and checks
commit 4d8a8ea711828325371b9ec351021b7739ea06ac Author: Iain R. Learmonth Date: Tue Aug 6 16:08:26 2019 +0100 Adds GitLab CI for tests and checks Configuration is held in the .gitlab-ci.yml file. Fixes: #31645 --- .gitlab-ci.yml | 18 ++ 1 file changed, 18 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000..7abea88 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,18 @@ +variables: + GIT_STRATEGY: clone + +stages: + - test + +test: + stage: test + image: debian:buster + script: + - apt update + - apt -y install default-jdk ant ivy git + - git submodule init + - git submodule update + - mkdir lib + - ant -lib /usr/share/java resolve + - ant test + - ant checks ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-lib/release] Use Ivy for resolving external dependencies.
commit e1728709b5ed30dace2dd5b7e47a3e3c5e9e5736 Author: Karsten Loesing Date: Sat Aug 3 11:57:02 2019 +0200 Use Ivy for resolving external dependencies. Includes an upgrade to the latest metrics-base commit 256e619. Implements part of #31326. --- CHANGELOG.md | 14 ++ build.xml| 3 ++- src/build| 2 +- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18d250e..560ee96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ +# Changes in version 2.7.0 - 2019-0?-?? + + * Medium changes + - Use Ivy for resolving external dependencies rather than relying + on files found in Debian stable packages. Requires installing Ivy + (using `apt-get install ivy`, `brew install ivy`, or similar) and + running `ant resolve` (or `ant -lib /usr/share/java resolve`). + Retrieved files are then copied to the `lib/` directory, except + for dependencies on other metrics libraries that still need to be + copied to the `lib/` directory manually. Current dependency + versions resolved by Ivy are the same as in Debian stretch with + few exceptions. + + # Changes in version 2.6.2 - 2019-05-29 * Medium changes diff --git a/build.xml b/build.xml index 3066e90..f319cf0 100644 --- a/build.xml +++ b/build.xml @@ -4,7 +4,8 @@ ]> - + diff --git a/src/build b/src/build index ed48192..256e619 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit ed4819215aa60cc8935f955d9404f30e83dd892b +Subproject commit 256e6192ad0066e48c6abb0ee9ac71714c7a3a0f ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-lib/release] Remove Cobertura from the build process.
commit 53b16d192da4a7fa5c1987a89a60257061602882 Author: Karsten Loesing Date: Mon Aug 12 09:34:04 2019 +0200 Remove Cobertura from the build process. --- CHANGELOG.md | 1 + src/build| 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 560ee96..b804ceb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ copied to the `lib/` directory manually. Current dependency versions resolved by Ivy are the same as in Debian stretch with few exceptions. + - Remove Cobertura from the build process. # Changes in version 2.6.2 - 2019-05-29 diff --git a/src/build b/src/build index 256e619..75cda35 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 256e6192ad0066e48c6abb0ee9ac71714c7a3a0f +Subproject commit 75cda35ba7375ab0f5b12fa214620c3a29917097 ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-lib/release] Revert all Debian buster library upgrades.
commit 7032e73a6471f3a4d7782ef65fc3d4cf00bb0ac1 Author: Karsten Loesing Date: Sun Jul 28 21:46:09 2019 +0200 Revert all Debian buster library upgrades. Turns out that updating all dependencies, including those in metrics-web and exonerator using servlets and JSPs, is much harder than expected. We decided to revert all these changes for now, so that we can merge patches again. We're going to investigate alternatives like Ant + Ivy, Maven, or Gradle in the near future. --- CHANGELOG.md | 20 build.xml| 14 +++--- src/build| 2 +- 3 files changed, 8 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e3622e1..18d250e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,23 +1,3 @@ -# Changes in version 2.7.0 - 2019-0?-?? - - * Medium changes - * Update build-time and run-time dependencies to versions found in - Debian buster, including: commons-codec-1.11.jar, - commons-compress-1.18.jar, jackson-annotations-2.9.8.jar, - jackson-core-2.9.8.jar, jackson-databind-2.9.8.jar, - logback-core-1.2.3.jar, logback-classic-1.2.3.jar, - slf4j-api-1.7.25.jar, and xz-1.8.jar. - * Update Checkstyle dependencies to versions found in Debian - buster, including: antlr-2.7.7.jar, antlr4-runtime-4.7.2.jar, - checkstyle-8.15.jar, commons-beanutils-1.9.3.jar, - commons-collections3-3.2.2.jar, commons-logging-1.2.jar, - guava-19.0.jar, java-atk-wrapper.jar, and picocli-3.9.2.jar. - * Update Cobertura dependencies to versions found in Debian buster, - including: asm4-5.0.4.jar, asm4-util-5.0.4.jar, - asm4-tree-5.0.4.jar, asm4-commons-5.0.4.jar, - asm4-analysis-5.0.4.jar, and commons-lang3-3.8.jar - - # Changes in version 2.6.2 - 2019-05-29 * Medium changes diff --git a/build.xml b/build.xml index a163dc9..3066e90 100644 --- a/build.xml +++ b/build.xml @@ -14,13 +14,13 @@ - - - - - - - + + + + + + + diff --git a/src/build b/src/build index 42ee1fc..ed48192 16 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 42ee1fc086b0e86cddf56475124a547465484321 +Subproject commit ed4819215aa60cc8935f955d9404f30e83dd892b ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-lib/release] Bump version to 2.6.2-dev.
commit e723c065b764ecfbb3bb96d4c491e67398b7f21b Author: Karsten Loesing Date: Thu May 30 15:21:41 2019 +0200 Bump version to 2.6.2-dev. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index e6a1dbe..3066e90 100644 --- a/build.xml +++ b/build.xml @@ -6,7 +6,7 @@ - + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-lib/master] Prepare for 2.7.0 release.
commit 0520779255dc44d30a69a25f24f5a638ae870a6e Author: Karsten Loesing Date: Fri Sep 6 08:53:14 2019 +0200 Prepare for 2.7.0 release. --- CHANGELOG.md | 2 +- build.xml| 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fdc45b..457533e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changes in version 2.7.0 - 2019-0?-?? +# Changes in version 2.7.0 - 2019-09-06 * Medium changes - Use Ivy for resolving external dependencies rather than relying diff --git a/build.xml b/build.xml index f319cf0..04f79fd 100644 --- a/build.xml +++ b/build.xml @@ -7,7 +7,7 @@ - + ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-lib/master] Add new SnowflakeStats descriptor type.
commit dc946fe1c8b50386f60f29a5073046eb2c0b27ad
Author: Karsten Loesing
Date: Tue Aug 13 15:01:45 2019 +0200
Add new SnowflakeStats descriptor type.
Implements #29461.
---
CHANGELOG.md | 1 +
.../org/torproject/descriptor/SnowflakeStats.java | 86 ++
.../descriptor/impl/DescriptorParserImpl.java | 5 +
.../java/org/torproject/descriptor/impl/Key.java | 6 +
.../torproject/descriptor/impl/ParseHelper.java| 21 +++
.../descriptor/impl/SnowflakeStatsImpl.java| 185 +
.../org/torproject/descriptor/DescriptorTest.java | 7 +-
.../descriptor/impl/SnowflakeStatsImplTest.java| 137 +++
src/test/resources/snowflake/example_metrics.log | 12 ++
9 files changed, 459 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b804ceb..2fdc45b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,7 @@
versions resolved by Ivy are the same as in Debian stretch with
few exceptions.
- Remove Cobertura from the build process.
+ - Add new SnowflakeStats descriptor type.
# Changes in version 2.6.2 - 2019-05-29
diff --git a/src/main/java/org/torproject/descriptor/SnowflakeStats.java
b/src/main/java/org/torproject/descriptor/SnowflakeStats.java
new file mode 100644
index 000..379a0f7
--- /dev/null
+++ b/src/main/java/org/torproject/descriptor/SnowflakeStats.java
@@ -0,0 +1,86 @@
+/* Copyright 2019 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.descriptor;
+
+import java.time.Duration;
+import java.time.LocalDateTime;
+import java.util.Optional;
+import java.util.SortedMap;
+
+/**
+ * Contain aggregated information about snowflake proxies and snowflake
clients.
+ *
+ * @since 2.7.0
+ */
+public interface SnowflakeStats extends Descriptor {
+
+ /**
+ * Return the end of the included measurement interval.
+ *
+ * @return End of the included measurement interval.
+ * @since 2.7.0
+ */
+ LocalDateTime snowflakeStatsEnd();
+
+ /**
+ * Return the length of the included measurement interval.
+ *
+ * @return Length of the included measurement interval.
+ * @since 2.7.0
+ */
+ Duration snowflakeStatsIntervalLength();
+
+ /**
+ * Return a list of mappings from two-letter country codes to the number of
+ * unique IP addresses of snowflake proxies that have polled.
+ *
+ * @return List of mappings from two-letter country codes to the number of
+ * unique IP addresses of snowflake proxies that have polled.
+ * @since 2.7.0
+ */
+ Optional> snowflakeIps();
+
+ /**
+ * Return a count of the total number of unique IP addresses of snowflake
+ * proxies that have polled.
+ *
+ * @return Count of the total number of unique IP addresses of snowflake
+ * proxies that have polled.
+ * @since 2.7.0
+ */
+ Optional snowflakeIpsTotal();
+
+ /**
+ * Return a count of the number of times a proxy has polled but received no
+ * client offer, rounded up to the nearest multiple of 8.
+ *
+ * @return Count of the number of times a proxy has polled but received no
+ * client offer, rounded up to the nearest multiple of 8.
+ * @since 2.7.0
+ */
+ Optional snowflakeIdleCount();
+
+ /**
+ * Return a count of the number of times a client has requested a proxy from
+ * the broker but no proxies were available, rounded up to the nearest
+ * multiple of 8.
+ *
+ * @return Count of the number of times a client has requested a proxy from
+ * the broker but no proxies were available, rounded up to the nearest
+ * multiple of 8.
+ * @since 2.7.0
+ */
+ Optional clientDeniedCount();
+
+ /**
+ * Return a count of the number of times a client successfully received a
+ * proxy from the broker, rounded up to the nearest multiple of 8.
+ *
+ * @return Count of the number of times a client successfully received a
proxy
+ * from the broker, rounded up to the nearest multiple of 8.
+ * @since 2.7.0
+ */
+ Optional clientSnowflakeMatchCount();
+}
+
diff --git
a/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java
b/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java
index ef9da56..9b620cb 100644
--- a/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java
@@ -130,6 +130,11 @@ public class DescriptorParserImpl implements
DescriptorParser {
} else if (firstLines.startsWith("@type torperf 1.")) {
return TorperfResultImpl.parseTorperfResults(rawDescriptorBytes,
sourceFile);
+} else if (firstLines.startsWith("@type snowflake-stats 1.")
+|| firstLines.startsWith(Key.SNOWFLAKE_STATS_END.keyword + SP)
+|| firstLines.contains(NL + Key.SNOWFLAKE_STATS_END.keyword + SP)) {
+ return this.parseOneOrMoreDescriptors(rawDesc
[tor-commits] [onionoo/master] Adds GitLab CI for tests and checks
commit 5173fe1d73c32342be937160d4974bd446c45e57 Author: Iain R. Learmonth Date: Thu Sep 5 12:26:45 2019 +0100 Adds GitLab CI for tests and checks Configuration is held in the .gitlab-ci.yml file. Fixes: #31400 --- .gitlab-ci.yml | 25 + 1 file changed, 25 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000..b67b5a4 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,25 @@ +variables: + GIT_STRATEGY: clone + METRICS_LIB_VERSION: "2.6.2" + JAVA_TOOL_OPTIONS: "-Dfile.encoding=UTF-8" + +stages: + - test + +test: + stage: test + image: debian:buster + script: + - apt update + - apt -y install default-jdk ant ivy git curl + - git submodule init + - git submodule update + - mkdir lib + - mkdir tmp + - pushd tmp + - curl https://dist.torproject.org/metrics-lib/$METRICS_LIB_VERSION/metrics-lib-$METRICS_LIB_VERSION.tar.gz | tar xzf - + - popd + - mv tmp/metrics-lib-$METRICS_LIB_VERSION/generated/dist/metrics-lib-$METRICS_LIB_VERSION-thin.jar lib + - ant -lib /usr/share/java resolve + - ant test + - ant checks ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [onionoo/master] Increase timeout for testCountryDeDe
commit 4ba7b00c6f4080955af907e1dbf9dd0c79fb9a58
Author: Iain R. Learmonth
Date: Thu Sep 5 14:10:44 2019 +0100
Increase timeout for testCountryDeDe
The timeout was being reached in the GitLab CI environment, which is not
super powerful.
---
.../java/org/torproject/metrics/onionoo/server/ResourceServletTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/src/test/java/org/torproject/metrics/onionoo/server/ResourceServletTest.java
b/src/test/java/org/torproject/metrics/onionoo/server/ResourceServletTest.java
index 7c3c52e..935c06e 100644
---
a/src/test/java/org/torproject/metrics/onionoo/server/ResourceServletTest.java
+++
b/src/test/java/org/torproject/metrics/onionoo/server/ResourceServletTest.java
@@ -1134,7 +1134,7 @@ public class ResourceServletTest {
null);
}
- @Test(timeout = 200)
+ @Test(timeout = 400)
public void testCountryDeDe() {
this.assertSummaryDocument(
"/summary?country=de=de", 1, new String[] { "TorkaZ" }, 0,
___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [metrics-lib/master] Adds GitLab CI for tests and checks
commit 4d8a8ea711828325371b9ec351021b7739ea06ac Author: Iain R. Learmonth Date: Tue Aug 6 16:08:26 2019 +0100 Adds GitLab CI for tests and checks Configuration is held in the .gitlab-ci.yml file. Fixes: #31645 --- .gitlab-ci.yml | 18 ++ 1 file changed, 18 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000..7abea88 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,18 @@ +variables: + GIT_STRATEGY: clone + +stages: + - test + +test: + stage: test + image: debian:buster + script: + - apt update + - apt -y install default-jdk ant ivy git + - git submodule init + - git submodule update + - mkdir lib + - ant -lib /usr/share/java resolve + - ant test + - ant checks ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
[tor-commits] [exonerator/master] Adds GitLab CI for tests and checks
commit 64782695e66679f485eeaa6030a80ec01598c25c Author: Iain R. Learmonth Date: Thu Sep 5 13:59:05 2019 +0100 Adds GitLab CI for tests and checks Configuration is held in the .gitlab-ci.yml file. Fixes: #31401 --- .gitlab-ci.yml | 25 + 1 file changed, 25 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000..b67b5a4 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,25 @@ +variables: + GIT_STRATEGY: clone + METRICS_LIB_VERSION: "2.6.2" + JAVA_TOOL_OPTIONS: "-Dfile.encoding=UTF-8" + +stages: + - test + +test: + stage: test + image: debian:buster + script: + - apt update + - apt -y install default-jdk ant ivy git curl + - git submodule init + - git submodule update + - mkdir lib + - mkdir tmp + - pushd tmp + - curl https://dist.torproject.org/metrics-lib/$METRICS_LIB_VERSION/metrics-lib-$METRICS_LIB_VERSION.tar.gz | tar xzf - + - popd + - mv tmp/metrics-lib-$METRICS_LIB_VERSION/generated/dist/metrics-lib-$METRICS_LIB_VERSION-thin.jar lib + - ant -lib /usr/share/java resolve + - ant test + - ant checks ___ tor-commits mailing list tor-commits@lists.torproject.org https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
