commit 5003567318b8cb694cb46462c4819b8cbcf2eaf4
Author: iwakeh <iwa...@torproject.org>
Date: Wed Oct 12 14:31:01 2016 +0200
Added property OutputPath and removed four obsolete path properties.
Added new methods to Configuration.
Implements part of task-20162 and prepares task-18910.
---
CHANGELOG.md | 7 ++++++
.../bridgedescs/SanitizedBridgesWriter.java | 19 +++++++++++----
.../torproject/collector/conf/Configuration.java | 27 +++++++++++++++++++++-
.../java/org/torproject/collector/conf/Key.java | 8 +++----
.../collector/exitlists/ExitListDownloader.java | 25 ++++++++++++--------
.../collector/relaydescs/ArchiveWriter.java | 5 ++--
.../collector/torperf/TorperfDownloader.java | 10 ++++----
src/main/resources/collector.properties | 15 ++++--------
.../bridgedescs/SanitizedBridgesWriterTest.java | 8 +++----
.../collector/conf/ConfigurationTest.java | 14 ++++++-----
10 files changed, 92 insertions(+), 46 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 33e77f6..fffad5e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,10 @@
+# Changes in version 1.1.0 - 2016-10-XX
+
+ * Medium changes
+ - Replace four properties for configuring where to write
+ descriptors by a single 'OutPath' property.
+
+
# Changes in version 1.0.2 - 2016-10-07
* Medium changes
diff --git
a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
index 6d82fc6..00aa54b 100644
---
a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
+++
b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
@@ -23,6 +23,7 @@ import java.io.FileWriter;
import java.io.IOException;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
+import java.nio.file.Paths;
import java.security.GeneralSecurityException;
import java.security.SecureRandom;
import java.text.ParseException;
@@ -53,6 +54,7 @@ public class SanitizedBridgesWriter extends CollecTorMain {
private static final Logger logger = LoggerFactory.getLogger(
SanitizedBridgesWriter.class);
+ private static final String BRIDGE_DESCRIPTORS = "bridge-descriptors";
public SanitizedBridgesWriter(Configuration config) {
super(config);
@@ -81,6 +83,10 @@ public class SanitizedBridgesWriter extends CollecTorMain {
private SecureRandom secureRandom;
+ private String outputPathName;
+
+ private String recentPathName;
+
@Override
public String module() {
return "bridgedescs";
@@ -89,10 +95,13 @@ public class SanitizedBridgesWriter extends CollecTorMain {
@Override
protected void startProcessing() throws ConfigurationException {
+ outputPathName = Paths.get(config.getPath(Key.OutputPath).toString(),
+ BRIDGE_DESCRIPTORS).toString();
+ recentPathName = Paths.get(config.getPath(Key.RecentPath).toString(),
+ BRIDGE_DESCRIPTORS).toString();
File bridgeDirectoriesDirectory =
config.getPath(Key.BridgeSnapshotsDirectory).toFile();
- File sanitizedBridgesDirectory =
- config.getPath(Key.SanitizedBridgesWriteDirectory).toFile();
+ File sanitizedBridgesDirectory = new File(outputPathName);
File statsDirectory = config.getPath(Key.StatsPath).toFile();
if (bridgeDirectoriesDirectory == null
@@ -593,8 +602,8 @@ public class SanitizedBridgesWriter extends CollecTorMain {
this.sanitizedBridgesDirectory.getAbsolutePath() + "/" + syear
+ "/" + smonth + "/statuses/" + sday + "/" + syear + smonth
+ sday + "-" + stime + "-" + authorityFingerprint);
- File rsyncFile = new File(config.getPath(Key.RecentPath).toFile(),
- "bridge-descriptors/statuses/" + tarballFile.getName());
+ File rsyncFile = new File(recentPathName, "statuses/"
+ + tarballFile.getName());
File[] outputFiles = new File[] { tarballFile, rsyncFile };
for (File outputFile : outputFiles) {
outputFile.getParentFile().mkdirs();
@@ -1373,7 +1382,7 @@ public class SanitizedBridgesWriter extends CollecTorMain
{
- 3L * 24L * 60L * 60L * 1000L;
Stack<File> allFiles = new Stack<File>();
allFiles.add(new File(config.getPath(Key.RecentPath).toFile(),
- "bridge-descriptors"));
+ BRIDGE_DESCRIPTORS));
while (!allFiles.isEmpty()) {
File file = allFiles.pop();
if (file.isDirectory()) {
diff --git a/src/main/java/org/torproject/collector/conf/Configuration.java
b/src/main/java/org/torproject/collector/conf/Configuration.java
index 52e3bad..b0b76ed 100644
--- a/src/main/java/org/torproject/collector/conf/Configuration.java
+++ b/src/main/java/org/torproject/collector/conf/Configuration.java
@@ -113,6 +113,11 @@ public class Configuration extends Observable implements
Cloneable {
return props.getProperty(key);
}
+ /** Retrieves the value for key returning a default for non-existing keys. */
+ public String getProperty(String key, String def) {
+ return props.getProperty(key, def);
+ }
+
/** Sets the value for key. */
public void setProperty(String key, String value) {
props.setProperty(key, value);
@@ -253,10 +258,30 @@ public class Configuration extends Observable implements
Cloneable {
try {
checkClass(key, URL.class);
return new URL(props.getProperty(key.name()));
- } catch (MalformedURLException mue) {
+ } catch (MalformedURLException | RuntimeException mue) {
throw new ConfigurationException("Corrupt property: " + key
+ " reason: " + mue.getMessage(), mue);
}
}
+ /**
+ * Returns {@code URL[]} from a property. Commas seperate array elements,
+ * e.g.,
+ * {@code propertyname = a1.example.org, a2.example2.com, a3.example3.net}
+ */
+ public URL[] getUrlArray(Key key) throws ConfigurationException {
+ try {
+ checkClass(key, URL[].class);
+ String[] interim = props.getProperty(key.name()).split(FIELDSEP);
+ URL[] res = new URL[interim.length];
+ for (int i = 0; i < interim.length; i++) {
+ res[i] = new URL(interim[i].trim());
+ }
+ return res;
+ } catch (MalformedURLException | RuntimeException re) {
+ throw new ConfigurationException("Corrupt property: " + key
+ + " reason: " + re.getMessage(), re);
+ }
+ }
+
}
diff --git a/src/main/java/org/torproject/collector/conf/Key.java
b/src/main/java/org/torproject/collector/conf/Key.java
index 5ec1be4..613dc7a 100644
--- a/src/main/java/org/torproject/collector/conf/Key.java
+++ b/src/main/java/org/torproject/collector/conf/Key.java
@@ -1,3 +1,6 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+
package org.torproject.collector.conf;
import java.net.URL;
@@ -11,11 +14,11 @@ public enum Key {
ShutdownGraceWaitMinutes(Long.class),
RunOnce(Boolean.class),
- ExitlistOutputDirectory(Path.class),
ExitlistUrl(URL.class),
InstanceBaseUrl(String.class),
ArchivePath(Path.class),
RecentPath(Path.class),
+ OutputPath(Path.class),
IndexPath(Path.class),
StatsPath(Path.class),
BridgedescsActivated(Boolean.class),
@@ -37,7 +40,6 @@ public enum Key {
CachedRelayDescriptorsDirectories(String[].class),
CompressRelayDescriptorDownloads(Boolean.class),
DirectoryArchivesDirectory(Path.class),
- DirectoryArchivesOutputDirectory(Path.class),
DownloadRelayDescriptors(Boolean.class),
DirectoryAuthoritiesAddresses(String[].class),
DirectoryAuthoritiesFingerprintsForVotes(String[].class),
@@ -54,8 +56,6 @@ public enum Key {
KeepDirectoryArchiveImportHistory(Boolean.class),
ReplaceIpAddressesWithHashes(Boolean.class),
BridgeDescriptorMappingsLimit(Integer.class),
- SanitizedBridgesWriteDirectory(Path.class),
- TorperfOutputDirectory(Path.class),
TorperfFilesLines(String[].class),
TorperfSources(String[][].class);
diff --git
a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java
b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java
index 2f71d2e..5609cea 100644
--- a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java
+++ b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java
@@ -23,6 +23,7 @@ import java.io.FileWriter;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
+import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
@@ -37,6 +38,12 @@ public class ExitListDownloader extends CollecTorMain {
private static final Logger logger = LoggerFactory.getLogger(
ExitListDownloader.class);
+ private static final String EXITLISTS = "exit-lists";
+
+ private String outputPathName;
+
+ private String recentPathName;
+
/** Instanciate the exit-lists module using the given configuration. */
public ExitListDownloader(Configuration config) {
super(config);
@@ -53,7 +60,10 @@ public class ExitListDownloader extends CollecTorMain {
SimpleDateFormat dateTimeFormat =
new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-
+ outputPathName = Paths.get(config.getPath(Key.OutputPath).toString(),
+ EXITLISTS).toString();
+ recentPathName = Paths.get(config.getPath(Key.RecentPath).toString(),
+ EXITLISTS).toString();
Date downloadedDate = new Date();
String downloadedExitList = null;
try {
@@ -94,9 +104,8 @@ public class ExitListDownloader extends CollecTorMain {
SimpleDateFormat tarballFormat =
new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
tarballFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- File tarballFile = new File(
- config.getPath(Key.ExitlistOutputDirectory).toFile(),
- tarballFormat.format(downloadedDate));
+ File tarballFile = Paths.get(outputPathName,
+ tarballFormat.format(downloadedDate)).toFile();
long maxScanMillis = 0L;
try {
@@ -128,8 +137,7 @@ public class ExitListDownloader extends CollecTorMain {
}
/* Write to disk. */
- File rsyncFile = new File(config.getPath(Key.RecentPath).toFile(),
- "exit-lists/" + tarballFile.getName());
+ File rsyncFile = new File(recentPathName, tarballFile.getName());
File[] outputFiles = new File[] { tarballFile, rsyncFile };
for (File outputFile : outputFiles) {
try {
@@ -148,7 +156,7 @@ public class ExitListDownloader extends CollecTorMain {
StringBuilder dumpStats = new StringBuilder("Finished downloading "
+ "exit list.\nLast three exit lists are:");
Stack<File> filesInInputDir = new Stack<File>();
- filesInInputDir.add(config.getPath(Key.ExitlistOutputDirectory).toFile());
+ filesInInputDir.add(new File(outputPathName));
SortedSet<File> lastThreeExitLists = new TreeSet<File>();
while (!filesInInputDir.isEmpty()) {
File pop = filesInInputDir.pop();
@@ -184,8 +192,7 @@ public class ExitListDownloader extends CollecTorMain {
long cutOffMillis = System.currentTimeMillis()
- 3L * 24L * 60L * 60L * 1000L;
Stack<File> allFiles = new Stack<File>();
- allFiles.add(new File(config.getPath(Key.RecentPath).toFile(),
- "/exit-lists"));
+ allFiles.add(new File(recentPathName, EXITLISTS));
while (!allFiles.isEmpty()) {
File file = allFiles.pop();
if (file.isDirectory()) {
diff --git
a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java
b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java
index 59b7969..cac5d14 100644
--- a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java
+++ b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java
@@ -129,8 +129,9 @@ public class ArchiveWriter extends CollecTorMain {
storedMicrodescriptorsFile
= new File(statsDir, "stored-microdescriptors");
File statsDirectory = config.getPath(Key.StatsPath).toFile();
- this.outputDirectory = config.getPath(Key.DirectoryArchivesOutputDirectory)
- .toString();
+ this.outputDirectory
+ = Paths.get(config.getPath(Key.OutputPath).toString(),
+ RELAY_DESCRIPTORS).toString();
SimpleDateFormat rsyncCatFormat = new SimpleDateFormat(
"yyyy-MM-dd-HH-mm-ss");
rsyncCatFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
diff --git
a/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java
b/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java
index 635c5a3..7c9367e 100644
--- a/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java
+++ b/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java
@@ -37,6 +37,8 @@ public class TorperfDownloader extends CollecTorMain {
private static final Logger logger = LoggerFactory.getLogger(
TorperfDownloader.class);
+ private static final String TORPERF = "torperf";
+
public TorperfDownloader(Configuration config) {
super(config);
}
@@ -49,14 +51,14 @@ public class TorperfDownloader extends CollecTorMain {
@Override
public String module() {
- return "torperf";
+ return TORPERF;
}
@Override
protected void startProcessing() throws ConfigurationException {
this.torperfFilesLines = config.getStringArray(Key.TorperfFilesLines);
- this.torperfOutputDirectory = config.getPath(Key.TorperfOutputDirectory)
- .toFile();
+ this.torperfOutputDirectory
+ = new File(config.getPath(Key.OutputPath).toString(), TORPERF);
this.torperfLastMergedFile = new
File(config.getPath(Key.StatsPath).toFile(),
"torperf-last-merged");
if (!this.torperfOutputDirectory.exists()) {
@@ -617,7 +619,7 @@ public class TorperfDownloader extends CollecTorMain {
long cutOffMillis = System.currentTimeMillis()
- 3L * 24L * 60L * 60L * 1000L;
Stack<File> allFiles = new Stack<File>();
- allFiles.add(new File(config.getPath(Key.RecentPath).toFile(), "torperf"));
+ allFiles.add(new File(config.getPath(Key.RecentPath).toFile(), TORPERF));
while (!allFiles.isEmpty()) {
File file = allFiles.pop();
if (file.isDirectory()) {
diff --git a/src/main/resources/collector.properties
b/src/main/resources/collector.properties
index 0c93cd0..1c02c92 100644
--- a/src/main/resources/collector.properties
+++ b/src/main/resources/collector.properties
@@ -59,6 +59,9 @@ ArchivePath = archive
# The top-level directory for the recent descriptors that were
# published in the last 72 hours.
RecentPath = recent
+# The top-level directory for the retrieved descriptors that will
+# be archived.
+OutputPath = out
# Some statistics are stored here.
StatsPath = stats
######## Relay descriptors ########
@@ -126,9 +129,6 @@ DownloadAllExtraInfoDescriptors = false
## Compress relay descriptors downloads by adding .z to the URLs
CompressRelayDescriptorDownloads = false
#
-## Path to directory to write directory archives to
-DirectoryArchivesOutputDirectory = out/relay-descriptors/
-#
#
######## Bridge descriptors ########
#
@@ -144,21 +144,14 @@ ReplaceIpAddressesWithHashes = false
## of days, or inf for unlimited.
BridgeDescriptorMappingsLimit = inf
#
-## Path to directory to write sanitized bridges to
-SanitizedBridgesWriteDirectory = out/bridge-descriptors/
-
+#
######## Exit lists ########
#
-##
-ExitlistOutputDirectory = out/exit-lists/
## Where to download exit-lists from.
ExitlistUrl = https://check.torproject.org/exit-addresses
######## Torperf downloader ########
#
-## Path to the directory to store Torperf files in.
-TorperfOutputDirectory = out/torperf/
-
## Torperf source names and base URLs
## multiple pairs can be specified separated by semi-colon, e.g.
## TorperfSourceName = torperf_A, http://some.torproject.org/; another,
http://another.torproject.org/
diff --git
a/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java
b/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java
index 7468b23..299bd92 100644
---
a/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java
+++
b/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java
@@ -101,7 +101,7 @@ public class SanitizedBridgesWriterTest {
this.recentDirectory = this.temporaryFolder.newFolder("recent");
this.statsDirectory = this.temporaryFolder.newFolder("stats").toString();
this.sanitizedBridgesDirectory =
- this.temporaryFolder.newFolder("out").toPath();
+ this.temporaryFolder.newFolder("out", "bridge-descriptors").toPath();
this.initializeTestConfiguration();
this.defaultServerDescriptorBuilder = new ServerDescriptorBuilder();
this.defaultExtraInfoDescriptorBuilder = new ExtraInfoDescriptorBuilder();
@@ -134,8 +134,8 @@ public class SanitizedBridgesWriterTest {
this.configuration.setProperty(Key.StatsPath.name(), statsDirectory);
this.configuration.setProperty(Key.BridgeSnapshotsDirectory.name(),
bridgeDirectoriesDir);
- this.configuration.setProperty(Key.SanitizedBridgesWriteDirectory.name(),
- sanitizedBridgesDirectory.toString());
+ this.configuration.setProperty(Key.OutputPath.name(),
+ sanitizedBridgesDirectory.toFile().getParent().toString());
}
/** Runs this test by executing all builders, performing the sanitizing
@@ -468,7 +468,7 @@ public class SanitizedBridgesWriterTest {
"router-digest B026CF0F81712D94BBF1362294882688DF247887");
assertEquals("Sanitized descriptor does not contain expected lines.",
expectedLines, this.parsedExtraInfoDescriptors.get(0));
- assertTrue("Sanitized descriptor file name differs.",
+ assertTrue("Sanitized descriptor file name differs. " + this.parsedFiles,
this.parsedFiles.containsKey("2016/06/extra-infos/b/0/"
+ "b026cf0f81712d94bbf1362294882688df247887"));
}
diff --git a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java
b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java
index fdba6b5..8e255c6 100644
--- a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java
+++ b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java
@@ -8,6 +8,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import org.torproject.collector.Main;
import org.torproject.collector.MainTest;
import org.torproject.collector.cron.CollecTorMain;
import org.torproject.collector.cron.Dummy;
@@ -19,6 +20,7 @@ import org.junit.rules.TemporaryFolder;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.lang.reflect.Field;
+import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Arrays;
@@ -38,16 +40,16 @@ public class ConfigurationTest {
public void testKeyCount() throws Exception {
assertEquals("The number of properties keys in enum Key changed."
+ "\n This test class should be adapted.",
- 49, Key.values().length);
+ 46, Key.values().length);
}
@Test()
public void testConfiguration() throws Exception {
Configuration conf = new Configuration();
String val = "xyz";
- conf.setProperty(Key.TorperfOutputDirectory.name(), val);
+ conf.setProperty(Key.OutputPath.name(), val);
assertEquals(1, conf.size());
- assertEquals(val, conf.getProperty(Key.TorperfOutputDirectory.name()));
+ assertEquals(val, conf.getProperty(Key.OutputPath.name()));
}
private String propLine(Key key, String val) {
@@ -108,9 +110,9 @@ public class ConfigurationTest {
Configuration conf = new Configuration();
for (String file : files) {
conf.clear();
- conf.setProperty(Key.DirectoryArchivesOutputDirectory.name(), file);
+ conf.setProperty(Key.OutputPath.name(), file);
assertEquals(new File(file),
- conf.getPath(Key.DirectoryArchivesOutputDirectory).toFile());
+ conf.getPath(Key.OutputPath).toFile());
}
}
@@ -131,7 +133,7 @@ public class ConfigurationTest {
public void testArrayArrayValueException() throws Exception {
Configuration conf = new Configuration();
conf.setProperty(Key.CachedRelayDescriptorsDirectories.name(), "");
- conf.getStringArrayArray(Key.TorperfOutputDirectory);
+ conf.getStringArrayArray(Key.OutputPath);
}
@Test(expected = ConfigurationException.class)
_______________________________________________
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
