commit 5da91cae110b4eba1e468250de8e08562562b06d
Author: Karsten Loesing <karsten.loes...@gmx.net>
Date:   Wed Sep 28 14:56:02 2016 +0200

    Add initial set of tests for bridgedescs module.
---
 .../collector/bridgedescs/DescriptorBuilder.java   |  77 +++
 .../bridgedescs/ExtraInfoDescriptorBuilder.java    |  55 +++
 .../bridgedescs/NetworkStatusBuilder.java          |  30 ++
 .../bridgedescs/SanitizedBridgesWriterTest.java    | 521 +++++++++++++++++++++
 .../bridgedescs/ServerDescriptorBuilder.java       |  74 +++
 .../collector/bridgedescs/TarballBuilder.java      | 108 +++++
 6 files changed, 865 insertions(+)

diff --git 
a/src/test/java/org/torproject/collector/bridgedescs/DescriptorBuilder.java 
b/src/test/java/org/torproject/collector/bridgedescs/DescriptorBuilder.java
new file mode 100644
index 0000000..bab8126
--- /dev/null
+++ b/src/test/java/org/torproject/collector/bridgedescs/DescriptorBuilder.java
@@ -0,0 +1,77 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.bridgedescs;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.List;
+
+/** Builds a descriptor by concatenating the given lines with newlines and
+ * writing the output to the given output stream. */
+abstract class DescriptorBuilder {
+
+  /** Descriptor lines. */
+  List<String> lines;
+
+  /** Removes all lines. */
+  void removeAllLines() {
+    this.lines.clear();
+  }
+
+  /** Removes the given line, or fails if that line cannot be found. */
+  void removeLine(String line) {
+    if (!this.lines.remove(line)) {
+      fail("Line not contained: " + line);
+    }
+  }
+
+  /** Removes all but the given line, or fails if that line cannot be found. */
+  void removeAllExcept(String line) {
+    assertTrue("Line not contained: " + line, this.lines.contains(line));
+    this.lines.retainAll(Arrays.asList(line));
+  }
+
+  /** Finds the first line that starts with the given line start and inserts 
the
+   * given lines before it, or fails if no line can be found with that line
+   * start. */
+  void insertBeforeLineStartingWith(String lineStart,
+      List<String> linesToInsert) {
+    for (int i = 0; i < this.lines.size(); i++) {
+      if (this.lines.get(i).startsWith(lineStart)) {
+        this.lines.addAll(i, linesToInsert);
+        return;
+      }
+    }
+    fail("Line start not found: " + lineStart);
+  }
+
+  /** Finds the first line that starts with the given line start and replaces
+   * that line and possibly subsequent lines, or fails if no line can be found
+   * with that line start or there are not enough lines left to replace. */
+  void replaceLineStartingWith(String lineStart, List<String> linesToReplace) {
+    for (int i = 0; i < this.lines.size(); i++) {
+      if (this.lines.get(i).startsWith(lineStart)) {
+        for (int j = 0; j < linesToReplace.size(); j++) {
+          assertTrue("Not enough lines left to replace.",
+              this.lines.size() > i + j);
+          this.lines.set(i + j, linesToReplace.get(j));
+        }
+        return;
+      }
+    }
+    fail("Line start not found: " + lineStart);
+  }
+
+  /** Writes all descriptor lines with newlines to the given output stream. */
+  void build(OutputStream outputStream) throws IOException {
+    for (String line : lines) {
+      outputStream.write((line + "\n").getBytes());
+    }
+  }
+}
+
diff --git 
a/src/test/java/org/torproject/collector/bridgedescs/ExtraInfoDescriptorBuilder.java
 
b/src/test/java/org/torproject/collector/bridgedescs/ExtraInfoDescriptorBuilder.java
new file mode 100644
index 0000000..a6ffa53
--- /dev/null
+++ 
b/src/test/java/org/torproject/collector/bridgedescs/ExtraInfoDescriptorBuilder.java
@@ -0,0 +1,55 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.bridgedescs;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+
+/** Builds a non-sanitized bridge extra-info descriptor that comes with an
+ * original bridge descriptor (of a bundled and therefore publicly known 
bridge)
+ * by default. */
+class ExtraInfoDescriptorBuilder extends DescriptorBuilder {
+
+  /** Initializes the descriptor builder. */
+  ExtraInfoDescriptorBuilder() {
+    this.lines = new ArrayList<>(Arrays.asList(
+        "extra-info MeekGoogle "
+            + "46D4A71197B8FA515A826C6B017C522FE264655B",
+        "identity-ed25519",
+        "-----BEGIN ED25519 CERT-----",
+        "AQQABjliAVz1pof1ijauJttRPRlhPc4GKgp7SWOtFsnvSA3ddIsIAQAgBAA6BoYk",
+        "ZEXE7RkiEJ1l5Ij9hc9TJOpM7/9XSPZnF/PbMfE0u3n3JbOO3s82GN6BPuA0v2Cs",
+        "5eSvciL7+38Ok2eCaMa6vDrXYUSKrN+z9Kz3feL/XDWQy9L9Tkm7bticng0=",
+        "-----END ED25519 CERT-----",
+        "published 2016-06-30 21:43:52",
+        "write-history 2016-06-30 18:40:48 (14400 s) "
+            + "415744,497664,359424,410624,420864,933888",
+        "read-history 2016-06-30 18:40:48 (14400 s) "
+            + "4789248,6237184,4473856,5039104,5567488,5440512",
+        "geoip-db-digest 6346E26E2BC96F8511588CE2695E9B0339A75D32",
+        "geoip6-db-digest 43CCB43DBC653D8CC16396A882C5F116A6004F0C",
+        "dirreq-stats-end 2016-06-30 14:40:48 (86400 s)",
+        "dirreq-v3-ips ",
+        "dirreq-v3-reqs ",
+        "dirreq-v3-resp ok=0,not-enough-sigs=0,unavailable=0,not-found=0,"
+            + "not-modified=0,busy=0",
+        "dirreq-v3-direct-dl complete=0,timeout=0,running=0",
+        "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0",
+        "transport meek 198.50.200.131:8000",
+        "transport meek 198.50.200.131:7443",
+        "bridge-stats-end 2016-06-30 14:41:18 (86400 s)",
+        "bridge-ips ",
+        "bridge-ip-versions v4=0,v6=0",
+        "bridge-ip-transports ",
+        "router-sig-ed25519 
xNkIgy3gYoENUDMvMvPj1/qPv4suyODE8PcVNLZpY8/WxKvoniT"
+            + "+2UsWvKsZVAZwFnq7kSByBJUGdxC3YdhSCA",
+        "router-signature",
+        "-----BEGIN SIGNATURE-----",
+        "jwfwSxul/olhO4VzJfBTg+KQf4G+nRwFa9XLMSgBTy6P+hqDkw7TE079BZiYb8+v",
+        "ElS08R1Diq50N8fosR5lqP/Ihhm+V0KcEyWG10+Vl7ADMA3m4GdbGa6dSrdiFMPs",
+        "OYE9aueVDIMgKyiOyNmgK3S8lwjX4v6yhaiJWxDGuKs=",
+        "-----END SIGNATURE-----"));
+  }
+}
+
diff --git 
a/src/test/java/org/torproject/collector/bridgedescs/NetworkStatusBuilder.java 
b/src/test/java/org/torproject/collector/bridgedescs/NetworkStatusBuilder.java
new file mode 100644
index 0000000..20088a4
--- /dev/null
+++ 
b/src/test/java/org/torproject/collector/bridgedescs/NetworkStatusBuilder.java
@@ -0,0 +1,30 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.bridgedescs;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+
+/** Builds a non-sanitized bridge network status that comes with an original
+ * bridge network status entry (of a bundled and therefore publicly known
+ * bridge) by default. */
+class NetworkStatusBuilder extends DescriptorBuilder {
+
+  /** Initializes the descriptor builder. */
+  NetworkStatusBuilder() {
+    this.lines = new ArrayList<>(Arrays.asList(
+        "published 2016-06-30 23:40:28",
+        "flag-thresholds stable-uptime=807660 stable-mtbf=1425164 "
+            + "fast-speed=47000 guard-wfu=98.000% guard-tk=691200 "
+            + "guard-bw-inc-exits=400000 guard-bw-exc-exits=402000 "
+            + "enough-mtbf=1 ignoring-advertised-bws=0",
+        "r MeekGoogle RtSnEZe4+lFagmxrAXxSL+JkZVs "
+            + "g+M7Ww+lGKmv6NW9GRmvzLOiR0Y 2016-06-30 21:43:52 "
+            + "198.50.200.131 8008 0",
+        "s Fast Running Stable Valid",
+        "w Bandwidth=56",
+        "p reject 1-65535"));
+  }
+}
+
diff --git 
a/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java
 
b/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java
new file mode 100644
index 0000000..0049bb6
--- /dev/null
+++ 
b/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java
@@ -0,0 +1,521 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.bridgedescs;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import org.torproject.collector.Main;
+import org.torproject.collector.conf.Configuration;
+import org.torproject.collector.conf.ConfigurationException;
+import org.torproject.collector.conf.Key;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/** Tests the bridge descriptor sanitizer by preparing a temporary folder
+ * with non-sanitized bridge descriptors, running the sanitizer, and
+ * verifying the sanitized descriptors. */
+public class SanitizedBridgesWriterTest {
+
+  /** Temporary folder containing all files for this test. */
+  @Rule
+  public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+  /** Directory containing bridge descriptor tarballs to sanitize. */
+  private File bridgeDirectoriesDir;
+
+  /** Directory holding recent descriptor files served by CollecTor. */
+  private File recentDirectory;
+
+  /** Directory storing all intermediate state that needs to be preserved
+   * between processing runs. */
+  private File statsDirectory;
+
+  /** Directory holding sanitized bridge descriptor files. */
+  private File sanitizedBridgesDirectory;
+
+  /** CollecTor configuration for this test. */
+  private Configuration configuration;
+
+  /** Server descriptor builder used to build the first and only server
+   * descriptor for this test, unless removed from the tarball builder.*/
+  private DescriptorBuilder defaultServerDescriptorBuilder;
+
+  /** Extra-info descriptor builder used to build the first and only
+   * extra-info descriptor for this test, unless removed from the tarball
+   * builder.*/
+  private DescriptorBuilder defaultExtraInfoDescriptorBuilder;
+
+  /** Network status builder used to build the first and only network
+   * status for this test, unless removed from the tarball builder.*/
+  private DescriptorBuilder defaultNetworkStatusBuilder;
+
+  /** Tarball builder to build the first and only tarball, unless removed
+   * from the test. */
+  private TarballBuilder defaultTarballBuilder;
+
+  /** Tarball builder(s) for this test. */
+  private List<TarballBuilder> tarballBuilders;
+
+  /** Parsed sanitized bridge descriptors with keys being file names and
+   * values being sanitized descriptor lines. */
+  private Map<String, List<String>> parsedFiles;
+
+  /** Parsed sanitized default server descriptor. */
+  private List<List<String>> parsedServerDescriptors;
+
+  /** Parsed sanitized default extra-info descriptor. */
+  private List<List<String>> parsedExtraInfoDescriptors;
+
+  /** Parsed sanitized default network status. */
+  private List<List<String>> parsedNetworkStatuses;
+
+  /** Prepares the temporary folder and the various builders for this
+   * test. */
+  @Before
+  public void createTemporaryFolderAndBuilders()
+      throws IOException {
+    this.bridgeDirectoriesDir = this.temporaryFolder.newFolder("in");
+    this.recentDirectory = this.temporaryFolder.newFolder("recent");
+    this.statsDirectory = this.temporaryFolder.newFolder("stats");
+    this.sanitizedBridgesDirectory = this.temporaryFolder.newFolder("out");
+    this.initializeTestConfiguration();
+    this.defaultServerDescriptorBuilder = new ServerDescriptorBuilder();
+    this.defaultExtraInfoDescriptorBuilder = new ExtraInfoDescriptorBuilder();
+    this.defaultNetworkStatusBuilder = new NetworkStatusBuilder();
+    this.defaultTarballBuilder = new TarballBuilder(
+        "from-tonga-2016-07-01T000702Z.tar.gz", 1467331624000L);
+    this.defaultTarballBuilder.add("bridge-descriptors", 1467331622000L,
+        Arrays.asList(new DescriptorBuilder[] {
+            this.defaultServerDescriptorBuilder }));
+    this.defaultTarballBuilder.add("cached-extrainfo", 1467327972000L,
+        Arrays.asList(new DescriptorBuilder[] {
+            this.defaultExtraInfoDescriptorBuilder }));
+    this.defaultTarballBuilder.add("cached-extrainfo.new", 1467331623000L,
+        Arrays.asList(new DescriptorBuilder[] { }));
+    this.defaultTarballBuilder.add("networkstatus-bridges",
+        1467330028000L, Arrays.asList(new DescriptorBuilder[] {
+            this.defaultNetworkStatusBuilder }));
+    this.tarballBuilders = new ArrayList<>(
+        Arrays.asList(this.defaultTarballBuilder));
+  }
+
+  /** Initializes a configuration for the bridge descriptor sanitizer. */
+  private void initializeTestConfiguration() throws IOException {
+    this.configuration = new Configuration();
+    this.configuration.load(getClass().getClassLoader().getResourceAsStream(
+        Main.CONF_FILE));
+    this.configuration.setProperty(Key.BridgedescsActivated.name(), "true");
+    this.configuration.setProperty(Key.RecentPath.name(),
+        recentDirectory.getAbsolutePath());
+    this.configuration.setProperty(Key.StatsPath.name(),
+        statsDirectory.getAbsolutePath());
+    this.configuration.setProperty(Key.BridgeSnapshotsDirectory.name(),
+        bridgeDirectoriesDir.getAbsolutePath());
+    this.configuration.setProperty(Key.SanitizedBridgesWriteDirectory.name(),
+        sanitizedBridgesDirectory.getAbsolutePath());
+  }
+
+  /** Runs this test by executing all builders, performing the sanitizing
+   * process, and parsing sanitized bridge descriptors for inspection. */
+  private void runTest() throws IOException, ConfigurationException {
+    for (TarballBuilder tarballBuilder : this.tarballBuilders) {
+      tarballBuilder.build(this.bridgeDirectoriesDir);
+    }
+    SanitizedBridgesWriter sbw = new SanitizedBridgesWriter(configuration);
+    sbw.startProcessing();
+    List<File> files = new ArrayList<>();
+    files.add(sanitizedBridgesDirectory);
+    String basePath = sanitizedBridgesDirectory.getAbsolutePath() + "/";
+    this.parsedFiles = new LinkedHashMap<>();
+    this.parsedServerDescriptors = new ArrayList<>();
+    this.parsedExtraInfoDescriptors = new ArrayList<>();
+    this.parsedNetworkStatuses = new ArrayList<>();
+    while (!files.isEmpty()) {
+      File file = files.remove(0);
+      if (file.isDirectory()) {
+        files.addAll(Arrays.asList(file.listFiles()));
+      } else {
+        List<String> parsedLines = new ArrayList<>();
+        BufferedReader reader = new BufferedReader(new FileReader(file));
+        String line;
+        while ((line = reader.readLine()) != null) {
+          parsedLines.add(line);
+        }
+        reader.close();
+        String relativePath = file.getAbsolutePath().substring(
+            basePath.length());
+        if (parsedLines.get(0).startsWith("@type bridge-server-descriptor ")) {
+          this.parsedServerDescriptors.add(parsedLines);
+        } else if (parsedLines.get(0).startsWith("@type bridge-extra-info ")) {
+          this.parsedExtraInfoDescriptors.add(parsedLines);
+        } else if (parsedLines.get(0).startsWith(
+            "@type bridge-network-status ")) {
+          this.parsedNetworkStatuses.add(parsedLines);
+        }
+        this.parsedFiles.put(relativePath, parsedLines);
+      }
+    }
+  }
+
+  @Test
+  public void testServerDescriptorDefault() throws Exception {
+    this.runTest();
+    List<String> expectedLines = Arrays.asList(
+        "@type bridge-server-descriptor 1.2",
+        "router MeekGoogle 127.0.0.1 1 0 0",
+        "master-key-ed25519 3HC9xnykNIfNdFDuJWwxaJSM5GCaIJKUtAYgMixbsOM",
+        "platform Tor 0.2.7.6 on Linux",
+        "protocols Link 1 2 Circuit 1",
+        "published 2016-06-30 21:43:52",
+        "fingerprint 88F7 4584 0F47 CE0C 6A4F E61D 8279 50B0 6F9E 4534",
+        "uptime 543754",
+        "bandwidth 3040870 5242880 56583",
+        "extra-info-digest B026CF0F81712D94BBF1362294882688DF247887 "
+            + "/XWPeILeik+uTGaKL3pnUeQfYS87SfjKVkwTiCmbqi0",
+        "hidden-service-dir",
+        "contact somebody",
+        "ntor-onion-key YjZG5eaQ1gmXvlSMGEBwM7OLswv8AtXZr6ccOnDUKQw=",
+        "reject *:*",
+        "router-digest-sha256 "
+            + "KI4PRYH9rDCDLYPNv9NF53gFy8pJjIpeJ7UkzIGOAnw",
+        "router-digest B6922FF5C045814DF4BCB72A0D6C9417CFFBD80A");
+    assertEquals("Sanitized descriptor does not contain expected lines.",
+        expectedLines, this.parsedServerDescriptors.get(0));
+    assertTrue("Sanitized descriptor file name differs.",
+        this.parsedFiles.containsKey("2016/06/server-descriptors/b/6/"
+        + "b6922ff5c045814df4bcb72a0d6c9417cffbd80a"));
+  }
+
+  @Test
+  public void testServerDescriptorEmpty() throws Exception {
+    this.defaultServerDescriptorBuilder.removeAllLines();
+    this.runTest();
+    assertTrue("No server descriptor provided as input.",
+        this.parsedServerDescriptors.isEmpty());
+  }
+
+  @Test
+  public void testServerDescriptorAdditionalAnnotation()
+      throws Exception {
+    this.defaultServerDescriptorBuilder.insertBeforeLineStartingWith(
+        "@purpose bridge", Arrays.asList("@source 198.50.200.131"));
+    this.runTest();
+    assertEquals("Expected 3 sanitized descriptors.", 3,
+        this.parsedFiles.size());
+  }
+
+  @Test
+  public void testServerDescriptorRouterLineTruncated() throws Exception {
+    this.defaultServerDescriptorBuilder.replaceLineStartingWith("router ",
+        Arrays.asList("router MeekGoogle"));
+    this.runTest();
+    assertTrue("Sanitized server descriptor with invalid router line.",
+        this.parsedServerDescriptors.isEmpty());
+  }
+
+  @Test
+  public void testServerDescriptorFingerprintTruncated() throws Exception {
+    this.defaultServerDescriptorBuilder.replaceLineStartingWith(
+        "fingerprint ", Arrays.asList("fingerprint 4"));
+    this.runTest();
+    assertTrue("Sanitized server descriptor with invalid fingerprint "
+        + "line.", this.parsedServerDescriptors.isEmpty());
+  }
+
+  @Test
+  public void testServerDescriptorFingerprintInvalidHex()
+      throws Exception {
+    this.defaultServerDescriptorBuilder.replaceLineStartingWith(
+        "fingerprint ", Arrays.asList("fingerprint FUN!"));
+    this.runTest();
+    assertTrue("Sanitized server descriptor with invalid fingerprint "
+        + "line.", this.parsedServerDescriptors.isEmpty());
+  }
+
+  @Test
+  public void testServerDescriptorFingerprintOpt() throws Exception {
+    this.defaultServerDescriptorBuilder.replaceLineStartingWith("fingerprint ",
+        Arrays.asList("opt fingerprint 46D4 A711 97B8 FA51 5A82 6C6B 017C 522F 
"
+        + "E264 655B"));
+    this.runTest();
+    this.parsedServerDescriptors.get(0).contains("opt fingerprint 88F7 "
+        + "4584 0F47 CE0C 6A4F E61D 8279 50B0 6F9E 4534");
+  }
+
+  @Test
+  public void testServerDescriptorExtraInfoDigestInvalidHex()
+      throws Exception {
+    this.defaultServerDescriptorBuilder.replaceLineStartingWith(
+        "extra-info-digest ", Arrays.asList("extra-info-digest 6"));
+    this.runTest();
+    assertTrue("Sanitized server descriptor with invalid extra-info "
+        + "line.", this.parsedServerDescriptors.isEmpty());
+  }
+
+  @Test
+  public void testServerDescriptorExtraInfoDigestSha1Only()
+      throws Exception {
+    this.defaultServerDescriptorBuilder.replaceLineStartingWith(
+        "extra-info-digest ", Arrays.asList("extra-info-digest "
+        + "6D03E80568DEFA102968D144CB35FFA6E3355B8A"));
+    this.runTest();
+    assertTrue("Expected different extra-info-digest line.",
+        this.parsedServerDescriptors.get(0).contains(
+        "extra-info-digest B026CF0F81712D94BBF1362294882688DF247887"));
+  }
+
+  @Test
+  public void testServerDescriptorExtraInfoDigestOpt() throws Exception {
+    this.defaultServerDescriptorBuilder.replaceLineStartingWith(
+        "extra-info-digest ", Arrays.asList("opt extra-info-digest "
+        + "6D03E80568DEFA102968D144CB35FFA6E3355B8A "
+        + "cy/LwP7nxukmmcT1+UnDg4qh0yKbjVUYKhGL8VksoJA"));
+    this.runTest();
+    this.parsedServerDescriptors.get(0).contains("opt extra-info-digest "
+        + "B026CF0F81712D94BBF1362294882688DF247887 "
+        + "/XWPeILeik+uTGaKL3pnUeQfYS87SfjKVkwTiCmbqi0");
+  }
+
+  @Test
+  public void testServerDescriptorRejectOwnAddress() throws Exception {
+    this.defaultServerDescriptorBuilder.insertBeforeLineStartingWith(
+        "reject *:*", Arrays.asList("reject 198.50.200.131:*", "accept *:80"));
+    this.runTest();
+    List<String> parsedLines = this.parsedServerDescriptors.get(0);
+    for (int i = 0; i < parsedLines.size(); i++) {
+      if ("reject 127.0.0.1:*".equals(parsedLines.get(i))) {
+        assertEquals("accept line out of order.", "accept *:80",
+            parsedLines.get(i + 1));
+        assertEquals("reject line out of order.", "reject *:*",
+            parsedLines.get(i + 2));
+        return;
+      }
+    }
+    fail("IP address in reject line should have been replaced: "
+        + parsedLines);
+  }
+
+  @Test
+  public void testServerDescriptorEd25519IdentityMasterKeyMismatch()
+      throws Exception {
+    this.defaultServerDescriptorBuilder.replaceLineStartingWith(
+        "master-key-ed25519 ", Arrays.asList("master-key-ed25519 "
+        + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"));
+    this.runTest();
+    assertTrue("Mismatch between Ed25519 identity and master key.",
+        this.parsedServerDescriptors.isEmpty());
+  }
+
+  @Test
+  public void testServerDescriptorFamilyInvalidFingerprint()
+      throws Exception {
+    this.defaultServerDescriptorBuilder.insertBeforeLineStartingWith(
+        "hidden-service-dir", Arrays.asList("family $0"));
+    this.runTest();
+    assertTrue("Sanitized server descriptor with invalid fingerprint in "
+        + "family line.", this.parsedServerDescriptors.isEmpty());
+  }
+
+  @Test
+  public void testExtraInfoDescriptorDefault() throws Exception {
+    this.runTest();
+    List<String> expectedLines = Arrays.asList(
+        "@type bridge-extra-info 1.3",
+        "extra-info MeekGoogle 88F745840F47CE0C6A4FE61D827950B06F9E4534",
+        "master-key-ed25519 3HC9xnykNIfNdFDuJWwxaJSM5GCaIJKUtAYgMixbsOM",
+        "published 2016-06-30 21:43:52",
+        "write-history 2016-06-30 18:40:48 (14400 s) "
+            + "415744,497664,359424,410624,420864,933888",
+        "read-history 2016-06-30 18:40:48 (14400 s) "
+            + "4789248,6237184,4473856,5039104,5567488,5440512",
+        "geoip-db-digest 6346E26E2BC96F8511588CE2695E9B0339A75D32",
+        "geoip6-db-digest 43CCB43DBC653D8CC16396A882C5F116A6004F0C",
+        "dirreq-stats-end 2016-06-30 14:40:48 (86400 s)",
+        "dirreq-v3-ips ",
+        "dirreq-v3-reqs ",
+        "dirreq-v3-resp ok=0,not-enough-sigs=0,unavailable=0,not-found=0,"
+            + "not-modified=0,busy=0",
+        "dirreq-v3-direct-dl complete=0,timeout=0,running=0",
+        "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0",
+        "transport meek",
+        "transport meek",
+        "bridge-stats-end 2016-06-30 14:41:18 (86400 s)",
+        "bridge-ips ",
+        "bridge-ip-versions v4=0,v6=0",
+        "bridge-ip-transports ",
+        "router-digest-sha256 "
+            + "/XWPeILeik+uTGaKL3pnUeQfYS87SfjKVkwTiCmbqi0",
+        "router-digest B026CF0F81712D94BBF1362294882688DF247887");
+    assertEquals("Sanitized descriptor does not contain expected lines.",
+        expectedLines, this.parsedExtraInfoDescriptors.get(0));
+    assertTrue("Sanitized descriptor file name differs.",
+        this.parsedFiles.containsKey("2016/06/extra-infos/b/0/"
+        + "b026cf0f81712d94bbf1362294882688df247887"));
+  }
+
+  @Test
+  public void testExtraInfoDescriptorExtraInfoInvalidHex()
+      throws Exception {
+    this.defaultExtraInfoDescriptorBuilder.replaceLineStartingWith(
+        "extra-info ", Arrays.asList("extra-info MeekGoogle 4"));
+    this.runTest();
+    assertTrue("Sanitized extra-info descriptor with invalid extra-info "
+        + "line.", this.parsedExtraInfoDescriptors.isEmpty());
+  }
+
+  @Test
+  public void testExtraInfoDescriptorRouterSignatureLineSpace()
+      throws Exception {
+    this.defaultExtraInfoDescriptorBuilder.replaceLineStartingWith(
+        "router-signature", Arrays.asList("router-signature "));
+    this.runTest();
+    assertTrue("Sanitized extra-info descriptor with invalid "
+        + "router-signature line.",
+        this.parsedExtraInfoDescriptors.isEmpty());
+  }
+
+  @Test
+  public void testNetworkStatusDefault() throws Exception {
+    this.runTest();
+    List<String> expectedLines = Arrays.asList(
+        "@type bridge-network-status 1.1",
+        "published 2016-06-30 23:40:28",
+        "flag-thresholds stable-uptime=807660 stable-mtbf=1425164 "
+            + "fast-speed=47000 guard-wfu=98.000% guard-tk=691200 "
+            + "guard-bw-inc-exits=400000 guard-bw-exc-exits=402000 "
+            + "enough-mtbf=1 ignoring-advertised-bws=0",
+        "r MeekGoogle iPdFhA9HzgxqT+YdgnlQsG+eRTQ "
+            + "tpIv9cBFgU30vLcqDWyUF8/72Ao 2016-06-30 21:43:52 127.0.0.1 "
+            + "1 0",
+        "s Fast Running Stable Valid",
+        "w Bandwidth=56",
+        "p reject 1-65535");
+    assertEquals("Sanitized descriptor does not contain expected lines.",
+        expectedLines, this.parsedNetworkStatuses.get(0));
+    assertTrue("Sanitized descriptor file name differs.",
+        this.parsedFiles.containsKey("2016/06/statuses/30/"
+        + "20160630-234028-4A0CCD2DDC7995083D73F5D667100C8A5831F16D"));
+  }
+
+  @Test
+  public void testNetworkStatusPublishedLineMissing() throws Exception {
+    this.defaultNetworkStatusBuilder.removeLine(
+        "published 2016-06-30 23:40:28");
+    this.runTest();
+    String sanitizedNetworkStatusFileName = "2016/07/statuses/01/"
+        + "20160701-000702-4A0CCD2DDC7995083D73F5D667100C8A5831F16D";
+    assertTrue("Sanitized descriptor file does contain published line.",
+        this.parsedFiles.get(sanitizedNetworkStatusFileName)
+        .contains("published 2016-07-01 00:07:02"));
+  }
+
+  @Test
+  public void testNetworkStatusPublishedLineMissingTarballFileNameChange()
+      throws Exception {
+    this.defaultNetworkStatusBuilder.removeLine(
+        "published 2016-06-30 23:40:28");
+    this.defaultTarballBuilder.setTarballFileName(
+        "from-tonga-with-love-2016-07-01T000702Z.tar.gz");
+    this.runTest();
+    assertTrue("Sanitized network status without published line and with "
+        + "changed tarball file name.", this.parsedNetworkStatuses.isEmpty());
+  }
+
+  @Test
+  public void testNetworkStatusAlinePortMissing() throws Exception {
+    this.configuration.setProperty(Key.ReplaceIpAddressesWithHashes.name(),
+        "true");
+    this.defaultNetworkStatusBuilder.insertBeforeLineStartingWith("s ",
+        Arrays.asList("a 198.50.200.132"));
+    this.runTest();
+    for (String line : this.parsedNetworkStatuses.get(0)) {
+      if (line.startsWith("a ")) {
+        fail("Sanitized a line without port: " + line);
+      }
+    }
+  }
+
+  @Test
+  public void testTarballContainsSameFileTwice() throws Exception {
+    this.defaultTarballBuilder.add("cached-extrainfo.new", 1467331623000L,
+        Arrays.asList(new DescriptorBuilder[] {
+            this.defaultExtraInfoDescriptorBuilder }));
+  }
+
+  @Test
+  public void testTarballCorrupt() throws Exception {
+    this.tarballBuilders.clear();
+    File tarballFile = new File(bridgeDirectoriesDir,
+        "from-tonga-2016-07-01T000702Z.tar.gz");
+    FileOutputStream stream = new FileOutputStream(tarballFile);
+    stream.write(new byte[] { 0x00 });
+    stream.close();
+    tarballFile.setLastModified(1467331624000L);
+    this.runTest();
+    assertTrue("Sanitized descriptors from corrupt tarball.",
+        this.parsedFiles.isEmpty());
+  }
+
+  @Test
+  public void testTarballUncompressed() throws Exception {
+    String tarballFileName = this.tarballBuilders.get(0).getTarballFileName();
+    this.tarballBuilders.get(0).setTarballFileName(
+        tarballFileName.substring(0, tarballFileName.length() - 3));
+    this.runTest();
+    assertEquals("Expected 3 sanitized descriptors from uncompressed "
+        + "tarball.", 3, this.parsedFiles.size());
+  }
+
+  @Test
+  public void testTarballBzip2Compressed() throws Exception {
+    String tarballFileName = this.tarballBuilders.get(0).getTarballFileName();
+    this.tarballBuilders.get(0).setTarballFileName(
+        tarballFileName.substring(0, tarballFileName.length() - 3) + ".bz2");
+    this.runTest();
+    assertTrue("Didn't expect sanitized descriptors from unsupported "
+        + "bz2-compressed tarball.", this.parsedFiles.isEmpty());
+  }
+
+  @Test
+  public void testParsedBridgeDirectoriesSkipTarball() throws Exception {
+    File parsedBridgeDirectoriesFile = new File(statsDirectory,
+        "parsed-bridge-directories");
+    BufferedWriter writer = new BufferedWriter(new FileWriter(
+        parsedBridgeDirectoriesFile));
+    writer.write(this.tarballBuilders.get(0).getTarballFileName() + "\n");
+    writer.close();
+    this.runTest();
+    assertTrue("Didn't expect sanitized descriptors after skipping "
+        + "tarball.", this.parsedFiles.isEmpty());
+  }
+
+  @Test
+  public void testParsedBridgeDirectoriesIsDirectory() throws Exception {
+    File parsedBridgeDirectoriesFile = new File(statsDirectory,
+        "parsed-bridge-directories");
+    parsedBridgeDirectoriesFile.mkdirs();
+    this.runTest();
+    assertTrue("Continued despite not being able to read "
+        + "parsed-bridge-directories.", this.parsedFiles.isEmpty());
+  }
+}
+
diff --git 
a/src/test/java/org/torproject/collector/bridgedescs/ServerDescriptorBuilder.java
 
b/src/test/java/org/torproject/collector/bridgedescs/ServerDescriptorBuilder.java
new file mode 100644
index 0000000..0b5e916
--- /dev/null
+++ 
b/src/test/java/org/torproject/collector/bridgedescs/ServerDescriptorBuilder.java
@@ -0,0 +1,74 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.bridgedescs;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+
+/** Builds a non-sanitized bridge server descriptor that comes with an original
+ * bridge descriptor (of a bundled and therefore publicly known bridge) by
+ * default. */
+class ServerDescriptorBuilder extends DescriptorBuilder {
+
+  /** Initializes the descriptor builder. */
+  ServerDescriptorBuilder() {
+    this.lines = new ArrayList<>(Arrays.asList(
+        "@purpose bridge",
+        "router MeekGoogle 198.50.200.131 8008 0 0",
+        "identity-ed25519",
+        "-----BEGIN ED25519 CERT-----",
+        "AQQABjliAVz1pof1ijauJttRPRlhPc4GKgp7SWOtFsnvSA3ddIsIAQAgBAA6BoYk",
+        "ZEXE7RkiEJ1l5Ij9hc9TJOpM7/9XSPZnF/PbMfE0u3n3JbOO3s82GN6BPuA0v2Cs",
+        "5eSvciL7+38Ok2eCaMa6vDrXYUSKrN+z9Kz3feL/XDWQy9L9Tkm7bticng0=",
+        "-----END ED25519 CERT-----",
+        "master-key-ed25519 "
+            + "OgaGJGRFxO0ZIhCdZeSI/YXPUyTqTO//V0j2Zxfz2zE",
+        "platform Tor 0.2.7.6 on Linux",
+        "protocols Link 1 2 Circuit 1",
+        "published 2016-06-30 21:43:52",
+        "fingerprint 46D4 A711 97B8 FA51 5A82 6C6B 017C 522F E264 655B",
+        "uptime 543754",
+        "bandwidth 3040870 5242880 56583",
+        "extra-info-digest 6D03E80568DEFA102968D144CB35FFA6E3355B8A "
+            + "cy/LwP7nxukmmcT1+UnDg4qh0yKbjVUYKhGL8VksoJA",
+        "onion-key",
+        "-----BEGIN RSA PUBLIC KEY-----",
+        "MIGJAoGBANcIfT+XV4HHSWEQPGkID0C4OgWQ3Gc/RmfQYLMPe5enDNSLBTstw4ep",
+        "aiScHB1xhN8xRhpVB/qaCcYGpmUltIH0NaWQ3tuRV7rw+fp7amfYZfThUk5OPpF0",
+        "soGd3jRrzX7SEm4YCGdLZALL51Wb2pdOmR93WucOZYav/tGs/d9rAgMBAAE=",
+        "-----END RSA PUBLIC KEY-----",
+        "signing-key",
+        "-----BEGIN RSA PUBLIC KEY-----",
+        "MIGJAoGBAMHXuK8J+5028rDovbEejPrsOJKWtsj7fr4EhMmOmIUM4N2gLdEVyFq7",
+        "sVkHZFf3v04PmOhSJymLmVVcXe+Qsb4U300DwADvnpeFjhU4trrFqZljZM5+gPW6",
+        "ZmK0ViD54td0biJZd9Ow65Od9XzbJTa2acO/sVXD0Q8tIfnEywvZAgMBAAE=",
+        "-----END RSA PUBLIC KEY-----",
+        "onion-key-crosscert",
+        "-----BEGIN CROSSCERT-----",
+        "zQvq4eQYXn9Y2St2Qch4AvwqPAJ+Y+MgTFTf4qYaQ04FXo1csf2eSPB5zbWaUgBb",
+        "GbtKaw1ZJJjEtVzk/HnIWQ/V/ONJUSL4BiF2M4RuhozJoK2BGpYfmcsGWQKeLcPi",
+        "YIVtO5OI2XcvxgGGVz4ZPPiiGDFJ2MxHA1747KnGSo8=",
+        "-----END CROSSCERT-----",
+        "ntor-onion-key-crosscert 1",
+        "-----BEGIN ED25519 CERT-----",
+        "AQoABjjOAToGhiRkRcTtGSIQnWXkiP2Fz1Mk6kzv/1dI9mcX89sxAA0280fSYhvB",
+        "Y39F6J5FuCFcE/B1KDZZP8zY3NYAP4y+jVTG82RRsN87hwZlyShoBxm2q3x4LNPl",
+        "67ZGbPdAUAA=",
+        "-----END ED25519 CERT-----",
+        "hidden-service-dir",
+        "contact jvictors at jessevictors com, PGP 0xC20BEC80, BTC "
+            + "1M6tuPXNmhbgSfaJqnxBUAf5tKi4TVhup8",
+        "ntor-onion-key YjZG5eaQ1gmXvlSMGEBwM7OLswv8AtXZr6ccOnDUKQw=",
+        "reject *:*",
+        "router-sig-ed25519 
ObNigP8q0HkRDaDPP84txWH+3TbuL8DUNLAF25OZV9uRovF9j02"
+            + "StDVEdLGR6vOx9lRos0/264n42jEHzmUbBg",
+        "router-signature",
+        "-----BEGIN SIGNATURE-----",
+        "u/J/T0w7JlH4yUbXcg5hDIVBzGZtXxoH+800zOJXIxbIEGqgTxOhA13C6s/j/C0G",
+        "+L6bcrNdqKriJERsJicT2UqVRiIl54c76J9ySsknNKvXuEbZ3RJ71FhzLbi5CQXJ",
+        "N5wdZX+AqHSnSe+ayaB3zVlp97gUbFhg3vE2eWPtRxY=",
+        "-----END SIGNATURE-----"));
+  }
+}
+
diff --git 
a/src/test/java/org/torproject/collector/bridgedescs/TarballBuilder.java 
b/src/test/java/org/torproject/collector/bridgedescs/TarballBuilder.java
new file mode 100644
index 0000000..eddfb2b
--- /dev/null
+++ b/src/test/java/org/torproject/collector/bridgedescs/TarballBuilder.java
@@ -0,0 +1,108 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.bridgedescs;
+
+import static org.junit.Assert.fail;
+
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import 
org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
+
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/** Builds a tarball containing non-sanitized bridge descriptors built using
+ * descriptor builders and writes the tarball to a new file with the given file
+ * name. */
+class TarballBuilder {
+
+  /** Internal helper class to store details about a file contained in the
+   * tarball. */
+  private class TarballFile {
+
+    /** Last modified time of the file. */
+    private long modifiedMillis;
+
+    /** Descriptor builders used to generate the file content. */
+    private List<DescriptorBuilder> descriptorBuilders;
+  }
+
+  /** File name of the tarball. */
+  private String tarballFileName;
+
+  String getTarballFileName() {
+    return tarballFileName;
+  }
+
+  void setTarballFileName(String tarballFileName) {
+    this.tarballFileName = tarballFileName;
+  }
+
+  /** Last modified time of the tarball file. */
+  private long modifiedMillis;
+
+  /** Files contained in the tarball. */
+  private Map<String, TarballFile> tarballFiles;
+
+  /** Initializes a new tarball builder that is going to write a tarball to the
+   * file with given file name and last-modified time. */
+  TarballBuilder(String tarballFileName, long modifiedMillis) {
+    this.tarballFileName = tarballFileName;
+    this.modifiedMillis = modifiedMillis;
+    this.tarballFiles = new LinkedHashMap<>();
+  }
+
+  /** Adds a new file to the tarball with given name, last-modified time, and
+   * descriptor builders to generate the file content. */
+  TarballBuilder add(String fileName, long modifiedMillis,
+      List<DescriptorBuilder> descriptorBuilders) throws IOException {
+    TarballFile file = new TarballFile();
+    file.modifiedMillis = modifiedMillis;
+    file.descriptorBuilders = descriptorBuilders;
+    this.tarballFiles.put(fileName, file);
+    return this;
+  }
+
+  /** Writes the previously configured tarball with all contained files to the
+   * given file, or fail if the file extension is not known. */
+  void build(File directory) throws IOException {
+    File tarballFile = new File(directory, this.tarballFileName);
+    TarArchiveOutputStream taos = null;
+    if (this.tarballFileName.endsWith(".tar.gz")) {
+      taos = new TarArchiveOutputStream(new GzipCompressorOutputStream(
+          new BufferedOutputStream(new FileOutputStream(tarballFile))));
+    } else if (this.tarballFileName.endsWith(".tar.bz2")) {
+      taos = new TarArchiveOutputStream(new BZip2CompressorOutputStream(
+          new BufferedOutputStream(new FileOutputStream(tarballFile))));
+    } else if (this.tarballFileName.endsWith(".tar")) {
+      taos = new TarArchiveOutputStream(new BufferedOutputStream(
+          new FileOutputStream(tarballFile)));
+    } else {
+      fail("Unknown file extension: " + this.tarballFileName);
+    }
+    for (Map.Entry<String, TarballFile> file : this.tarballFiles.entrySet()) {
+      TarArchiveEntry tae = new TarArchiveEntry(file.getKey());
+      ByteArrayOutputStream baos = new ByteArrayOutputStream();
+      for (DescriptorBuilder descriptorBuilder
+          : file.getValue().descriptorBuilders) {
+        descriptorBuilder.build(baos);
+      }
+      tae.setSize(baos.size());
+      tae.setModTime(file.getValue().modifiedMillis);
+      taos.putArchiveEntry(tae);
+      taos.write(baos.toByteArray());
+      taos.closeArchiveEntry();
+    }
+    taos.close();
+    tarballFile.setLastModified(this.modifiedMillis);
+  }
+}
+



_______________________________________________
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

Reply via email to