commit 3121a2692cf76c344c70a1bf2abcaed4966495eb
Author: iwakeh <iwa...@torproject.org>
Date:   Wed Oct 12 19:08:04 2016 +0200

    Introduced *Sources and *Origins properties to simplify data
    source definition.
    Already provides properties for task-18910.
---
 CHANGELOG.md                                       |  2 +
 .../bridgedescs/SanitizedBridgesWriter.java        |  2 +-
 .../torproject/collector/conf/Configuration.java   | 26 ++++++++++
 .../java/org/torproject/collector/conf/Key.java    | 16 +++---
 .../org/torproject/collector/conf/SourceType.java  | 11 +++++
 .../collector/relaydescs/ArchiveWriter.java        | 13 +++--
 src/main/resources/collector.properties            | 39 +++++++++------
 .../bridgedescs/SanitizedBridgesWriterTest.java    |  2 +-
 .../collector/conf/ConfigurationTest.java          | 57 +++++++++++++++++-----
 9 files changed, 129 insertions(+), 39 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index fffad5e..6e1b7eb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,8 @@
  * Medium changes
    - Replace four properties for configuring where to write
      descriptors by a single 'OutPath' property.
+   - Introduced *Sources and *Origins properties to simplify
+     data source definition.
 
 
 # Changes in version 1.0.2 - 2016-10-07
diff --git 
a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
 
b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
index 00aa54b..79cb13c 100644
--- 
a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
+++ 
b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
@@ -100,7 +100,7 @@ public class SanitizedBridgesWriter extends CollecTorMain {
     recentPathName = Paths.get(config.getPath(Key.RecentPath).toString(),
         BRIDGE_DESCRIPTORS).toString();
     File bridgeDirectoriesDirectory =
-        config.getPath(Key.BridgeSnapshotsDirectory).toFile();
+        config.getPath(Key.BridgeLocalOrigins).toFile();
     File sanitizedBridgesDirectory = new File(outputPathName);
     File statsDirectory = config.getPath(Key.StatsPath).toFile();
 
diff --git a/src/main/java/org/torproject/collector/conf/Configuration.java 
b/src/main/java/org/torproject/collector/conf/Configuration.java
index b0b76ed..b63a00c 100644
--- a/src/main/java/org/torproject/collector/conf/Configuration.java
+++ b/src/main/java/org/torproject/collector/conf/Configuration.java
@@ -15,8 +15,10 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.attribute.FileTime;
+import java.util.EnumSet;
 import java.util.Observable;
 import java.util.Properties;
+import java.util.Set;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
@@ -251,6 +253,30 @@ public class Configuration extends Observable implements 
Cloneable {
   }
 
   /**
+   * Returns a {@code SourceType} as List, e.g.
+   * {@code sourcetypeproperty = Remote, Sync}.
+   */
+  public Set<SourceType> getSourceTypeSet(Key key) throws 
ConfigurationException {
+    Set<SourceType> res = null;
+    try {
+      checkClass(key, SourceType[].class);
+      String[] interim = props.getProperty(key.name()).split(FIELDSEP);
+      for (int i = 0; i < interim.length; i++) {
+        SourceType st = SourceType.valueOf(interim[i].trim());
+        if (null == res) {
+          res = EnumSet.of(st);
+        } else {
+          res.add(st);
+        }
+      }
+      return res;
+    } catch (RuntimeException re) {
+      throw new ConfigurationException("Corrupt property: " + key
+          + " reason: " + re.getMessage(), re);
+    }
+  }
+
+  /**
    * Returns a {@code URL} property, e.g.
    * {@code urlProperty = https://my.url.here}.
    */
diff --git a/src/main/java/org/torproject/collector/conf/Key.java 
b/src/main/java/org/torproject/collector/conf/Key.java
index 613dc7a..d06fe0c 100644
--- a/src/main/java/org/torproject/collector/conf/Key.java
+++ b/src/main/java/org/torproject/collector/conf/Key.java
@@ -21,6 +21,16 @@ public enum Key {
   OutputPath(Path.class),
   IndexPath(Path.class),
   StatsPath(Path.class),
+  SyncPath(Path.class),
+  RelaySources(SourceType[].class),
+  BridgeSources(SourceType[].class),
+  ExitlistSources(SourceType[].class),
+  RelayCacheOrigins(String[].class),
+  RelayLocalOrigins(Path.class),
+  RelaySyncOrigins(URL[].class),
+  BridgeSyncOrigins(URL[].class),
+  BridgeLocalOrigins(Path.class),
+  ExitlistSyncOrigins(URL[].class),
   BridgedescsActivated(Boolean.class),
   BridgedescsOffsetMinutes(Integer.class),
   BridgedescsPeriodMinutes(Integer.class),
@@ -36,11 +46,7 @@ public enum Key {
   UpdateindexActivated(Boolean.class),
   UpdateindexOffsetMinutes(Integer.class),
   UpdateindexPeriodMinutes(Integer.class),
-  BridgeSnapshotsDirectory(Path.class),
-  CachedRelayDescriptorsDirectories(String[].class),
   CompressRelayDescriptorDownloads(Boolean.class),
-  DirectoryArchivesDirectory(Path.class),
-  DownloadRelayDescriptors(Boolean.class),
   DirectoryAuthoritiesAddresses(String[].class),
   DirectoryAuthoritiesFingerprintsForVotes(String[].class),
   DownloadCurrentConsensus(Boolean.class),
@@ -51,8 +57,6 @@ public enum Key {
   DownloadMissingMicrodescriptors(Boolean.class),
   DownloadAllServerDescriptors(Boolean.class),
   DownloadAllExtraInfoDescriptors(Boolean.class),
-  ImportCachedRelayDescriptors(Boolean.class),
-  ImportDirectoryArchives(Boolean.class),
   KeepDirectoryArchiveImportHistory(Boolean.class),
   ReplaceIpAddressesWithHashes(Boolean.class),
   BridgeDescriptorMappingsLimit(Integer.class),
diff --git a/src/main/java/org/torproject/collector/conf/SourceType.java 
b/src/main/java/org/torproject/collector/conf/SourceType.java
new file mode 100644
index 0000000..b6eb7ce
--- /dev/null
+++ b/src/main/java/org/torproject/collector/conf/SourceType.java
@@ -0,0 +1,11 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.conf;
+
+public enum SourceType {
+  Cache,
+  Local,
+  Remote,
+  Sync;
+}
diff --git 
a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java 
b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java
index cac5d14..cdf6506 100644
--- a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java
+++ b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java
@@ -6,6 +6,7 @@ package org.torproject.collector.relaydescs;
 import org.torproject.collector.conf.Configuration;
 import org.torproject.collector.conf.ConfigurationException;
 import org.torproject.collector.conf.Key;
+import org.torproject.collector.conf.SourceType;
 import org.torproject.collector.cron.CollecTorMain;
 import org.torproject.descriptor.DescriptorParseException;
 import org.torproject.descriptor.DescriptorParser;
@@ -146,7 +147,9 @@ public class ArchiveWriter extends CollecTorMain {
     RelayDescriptorParser rdp = new RelayDescriptorParser(this);
 
     RelayDescriptorDownloader rdd = null;
-    if (config.getBool(Key.DownloadRelayDescriptors)) {
+
+    Set<SourceType> sources = config.getSourceTypeSet(Key.RelaySources);
+    if (sources.contains(SourceType.Remote)) {
       String[] dirSources =
           config.getStringArray(Key.DirectoryAuthoritiesAddresses);
       rdd = new RelayDescriptorDownloader(rdp, dirSources,
@@ -162,15 +165,15 @@ public class ArchiveWriter extends CollecTorMain {
           config.getBool(Key.CompressRelayDescriptorDownloads));
       rdp.setRelayDescriptorDownloader(rdd);
     }
-    if (config.getBool(Key.ImportCachedRelayDescriptors)) {
+    if (sources.contains(SourceType.Cache)) {
       new CachedRelayDescriptorReader(rdp,
-          config.getStringArray(Key.CachedRelayDescriptorsDirectories), 
statsDirectory);
+          config.getStringArray(Key.RelayCacheOrigins), statsDirectory);
       this.intermediateStats("importing relay descriptors from local "
           + "Tor data directories");
     }
-    if (config.getBool(Key.ImportDirectoryArchives)) {
+    if (sources.contains(SourceType.Local)) {
       new ArchiveReader(rdp,
-                        
config.getPath(Key.DirectoryArchivesDirectory).toFile(),
+          config.getPath(Key.RelayLocalOrigins).toFile(),
           statsDirectory,
           config.getBool(Key.KeepDirectoryArchiveImportHistory));
       this.intermediateStats("importing relay descriptors from local "
diff --git a/src/main/resources/collector.properties 
b/src/main/resources/collector.properties
index 1c02c92..4441e1d 100644
--- a/src/main/resources/collector.properties
+++ b/src/main/resources/collector.properties
@@ -64,20 +64,22 @@ RecentPath = recent
 OutputPath = out
 # Some statistics are stored here.
 StatsPath = stats
+# Path for descriptors downloaded from other instances
+SyncPath = sync
 ######## Relay descriptors ########
 #
-## Read cached-* files from a local Tor data directory
-ImportCachedRelayDescriptors = false
+## Define descriptor sources
+#  possible values: Sync, Cache, Remote, Local
+RelaySources = Remote
+#  Retrieve files from the following CollecTor instances.
+#  List of URLs separated by comma.
+RelaySyncOrigins = https://collector.torproject.org
 #
-## Relative path to Tor data directory to read cached-* files from
+## Path to Tor data directory to read cached-* files from
 ## the listed path(s). If there is more that one separated by comma.
-CachedRelayDescriptorsDirectories = in/relay-descriptors/cacheddesc/
-#
-## Import directory archives from disk, if available
-ImportDirectoryArchives = false
-#
+RelayCacheOrigins = in/relay-descriptors/cacheddesc/
 ## Relative path to directory to import directory archives from
-DirectoryArchivesDirectory = in/relay-descriptors/archives/
+RelayLocalOrigins = in/relay-descriptors/archives/
 #
 ## Keep a history of imported directory archive files to know which files
 ## have been imported before. This history can be useful when importing
@@ -85,9 +87,6 @@ DirectoryArchivesDirectory = in/relay-descriptors/archives/
 ## again, but it can be confusing to users who don't know about it.
 KeepDirectoryArchiveImportHistory = false
 #
-## Download relay descriptors from directory authorities, if required
-DownloadRelayDescriptors = false
-#
 ## Comma separated list of directory authority addresses (IP[:port]) to
 ## download missing relay descriptors from
 DirectoryAuthoritiesAddresses = 
128.31.0.39:9131,86.59.21.38,194.109.206.212,131.188.40.189,193.23.244.244,171.25.193.9:443,154.35.175.225,199.254.238.52
@@ -132,8 +131,14 @@ CompressRelayDescriptorDownloads = false
 #
 ######## Bridge descriptors ########
 #
+## Define descriptor sources
+#  possible values: Sync, Local
+BridgeSources = Local
+#  Retrieve files from the following instances.
+#  List of URLs separated by comma.
+BridgeSyncOrigins = https://collector.torproject.org
 ## Relative path to directory to import bridge descriptor snapshots from
-BridgeSnapshotsDirectory = in/bridge-descriptors/
+BridgeLocalOrigins = in/bridge-descriptors/
 #
 ## Replace IP addresses in sanitized bridge descriptors with 10.x.y.z
 ## where x.y.z = H(IP address | bridge identity | secret)[:3], so that we
@@ -147,9 +152,15 @@ BridgeDescriptorMappingsLimit = inf
 #
 ######## Exit lists ########
 #
+## Define descriptor sources
+#  possible values: Sync, Remote
+ExitlistSources = Remote
+##  Retrieve files from the following instances.
+##  List of URLs separated by comma.
+ExitlistSyncOrigins = https://collector.torproject.org
 ## Where to download exit-lists from.
 ExitlistUrl = https://check.torproject.org/exit-addresses
-
+#
 ######## Torperf downloader ########
 #
 ## Torperf source names and base URLs
diff --git 
a/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java
 
b/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java
index 299bd92..947f486 100644
--- 
a/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java
+++ 
b/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java
@@ -132,7 +132,7 @@ public class SanitizedBridgesWriterTest {
     this.configuration.setProperty(Key.RecentPath.name(),
         recentDirectory.getAbsolutePath());
     this.configuration.setProperty(Key.StatsPath.name(), statsDirectory);
-    this.configuration.setProperty(Key.BridgeSnapshotsDirectory.name(),
+    this.configuration.setProperty(Key.BridgeLocalOrigins.name(),
         bridgeDirectoriesDir);
     this.configuration.setProperty(Key.OutputPath.name(),
         sanitizedBridgesDirectory.toFile().getParent().toString());
diff --git a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java 
b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java
index 8e255c6..3f9c231 100644
--- a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java
+++ b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java
@@ -27,6 +27,7 @@ import java.util.Arrays;
 import java.util.Observable;
 import java.util.Observer;
 import java.util.Random;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 public class ConfigurationTest {
@@ -40,7 +41,7 @@ public class ConfigurationTest {
   public void testKeyCount() throws Exception {
     assertEquals("The number of properties keys in enum Key changed."
         + "\n This test class should be adapted.",
-        46, Key.values().length);
+        50, Key.values().length);
   }
 
   @Test()
@@ -70,11 +71,11 @@ public class ConfigurationTest {
     Configuration conf = new Configuration();
     for (String input : arrays) {
       conf.clear();
-      conf.setProperty(Key.CachedRelayDescriptorsDirectories.name(), input);
+      conf.setProperty(Key.RelayCacheOrigins.name(), input);
       assertArrayEquals("expected " + Arrays.toString(array) + "\nreceived: "
           + Arrays.toString(conf
-              .getStringArray(Key.CachedRelayDescriptorsDirectories)),
-          array, conf.getStringArray(Key.CachedRelayDescriptorsDirectories));
+              .getStringArray(Key.RelayCacheOrigins)),
+          array, conf.getStringArray(Key.RelayCacheOrigins));
     }
   }
 
@@ -82,10 +83,10 @@ public class ConfigurationTest {
   public void testBoolValues() throws Exception {
     Configuration conf = new Configuration();
     conf.setProperty(Key.CompressRelayDescriptorDownloads.name(), "false");
-    conf.setProperty(Key.ImportDirectoryArchives.name(), "trUe");
+    conf.setProperty(Key.RunOnce.name(), "trUe");
     conf.setProperty(Key.ReplaceIpAddressesWithHashes.name(), "false");
     assertFalse(conf.getBool(Key.CompressRelayDescriptorDownloads));
-    assertTrue(conf.getBool(Key.ImportDirectoryArchives));
+    assertTrue(conf.getBool(Key.RunOnce));
     assertFalse(conf.getBool(Key.ReplaceIpAddressesWithHashes));
   }
 
@@ -117,6 +118,19 @@ public class ConfigurationTest {
   }
 
   @Test()
+  public void testSourceTypeValues() throws Exception {
+    String[] types = new String[] { "Local", "Cache", "Remote", "Sync"};
+    Configuration conf = new Configuration();
+    for (String type : types) {
+      conf.clear();
+      conf.setProperty(Key.BridgeSources.name(), type);
+      Set<SourceType> sts = conf.getSourceTypeSet(Key.BridgeSources);
+      assertEquals(1, sts.size());
+      assertTrue(sts.contains(SourceType.valueOf(type)));
+    }
+  }
+
+  @Test()
   public void testArrayArrayValues() throws Exception {
     String[][] sourceStrings = new String[][] {
       new String[]{"localsource", "http://127.0.0.1:12345"},
@@ -129,17 +143,36 @@ public class ConfigurationTest {
         conf.getStringArrayArray(Key.TorperfSources));
   }
 
+  @Test()
+  public void testUrlArrayValues() throws Exception {
+    URL[] array = new URL[randomSource.nextInt(30) + 1];
+    for (int i = 0; i < array.length; i++) {
+      array[i] = new URL("https://";
+          + Integer.toBinaryString(randomSource.nextInt(100)) + ".dummy.org");
+    }
+    String input =
+        Arrays.toString(array).replace("[", "").replace("]", "")
+            .replaceAll(" ", "");
+    Configuration conf = new Configuration();
+    conf.clear();
+    conf.setProperty(Key.RelaySyncOrigins.name(), input);
+    assertArrayEquals("expected " + Arrays.toString(array) + "\nreceived: "
+        + Arrays.toString(conf
+            .getUrlArray(Key.RelaySyncOrigins)),
+        array, conf.getUrlArray(Key.RelaySyncOrigins));
+  }
+
   @Test(expected = ConfigurationException.class)
   public void testArrayArrayValueException() throws Exception {
     Configuration conf = new Configuration();
-    conf.setProperty(Key.CachedRelayDescriptorsDirectories.name(), "");
+    conf.setProperty(Key.RelayCacheOrigins.name(), "");
     conf.getStringArrayArray(Key.OutputPath);
   }
 
   @Test(expected = ConfigurationException.class)
   public void testArrayValueException() throws Exception {
     Configuration conf = new Configuration();
-    conf.setProperty(Key.CachedRelayDescriptorsDirectories.name(), "");
+    conf.setProperty(Key.RelayCacheOrigins.name(), "");
     conf.getStringArray(Key.TorperfSources);
   }
 
@@ -147,14 +180,14 @@ public class ConfigurationTest {
   public void testBoolValueException() throws Exception {
     Configuration conf = new Configuration();
     conf.setProperty(Key.TorperfSources.name(), "http://x.y.z";);
-    conf.getBool(Key.CachedRelayDescriptorsDirectories);
+    conf.getBool(Key.RelayCacheOrigins);
   }
 
   @Test(expected = ConfigurationException.class)
   public void testPathValueException() throws Exception {
     Configuration conf = new Configuration();
-    conf.setProperty(Key.DirectoryArchivesDirectory.name(), "\\\u0000:");
-    conf.getPath(Key.DirectoryArchivesDirectory);
+    conf.setProperty(Key.RelayLocalOrigins.name(), "\\\u0000:");
+    conf.getPath(Key.RelayLocalOrigins);
   }
 
   @Test(expected = ConfigurationException.class)
@@ -214,7 +247,7 @@ public class ConfigurationTest {
     conf.setWatchableSourceAndLoad(confFile.toPath());
     MainTest.waitSec(1);
     confFile.delete();
-    conf.setProperty(Key.ImportDirectoryArchives.name(), "false");
+    conf.setProperty(Key.RunOnce.name(), "false");
     final Dummy dummy = new Dummy(conf);
     tmpf.newFolder("empty");
     MainTest.waitSec(6);

_______________________________________________
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

Reply via email to