This is an automated email from the ASF dual-hosted git repository.
lewismc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nutch.git
The following commit(s) were added to refs/heads/master by this push:
new 35734ec8b NUTCH-3249 CommonCrawlDataDumper should use safe names for
archive entries built from crawled content (#994)
35734ec8b is described below
commit 35734ec8b1a42b7bf4f802e8aa604ca1d58630b6
Author: Lewis John McGibbney <[email protected]>
AuthorDate: Wed Nov 11 05:19:08 2026 -0800
NUTCH-3249 CommonCrawlDataDumper should use safe names for archive entries
built from crawled content (#994)
---
.../apache/nutch/tools/CommonCrawlDataDumper.java | 20 ++-
.../nutch/tools/TestCommonCrawlDataDumper.java | 143 +++++++++++++++++++++
2 files changed, 162 insertions(+), 1 deletion(-)
diff --git a/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
b/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
index 2bba97918..31b892510 100644
--- a/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
+++ b/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
@@ -334,7 +334,8 @@ public class CommonCrawlDataDumper extends NutchTool
implements Tool {
reverseKey, url, timestamp, extensionName, !gzip);
outputRelativePath = outputFullPath
.substring(0, outputFullPath.lastIndexOf(File.separator) -
1);
- filename = content.getMetadata().get(Metadata.DATE) + "."
+ filename = sanitizeArchiveFileName(
+ content.getMetadata().get(Metadata.DATE)) + "."
+ extensionName;
} else {
String md5Ofurl = DumpFileUtil.getUrlMD5(url);
@@ -530,6 +531,23 @@ public class CommonCrawlDataDumper extends NutchTool
implements Tool {
typeCounts.containsKey(mimeType) ? typeCounts.get(mimeType) + 1 : 1);
}
+ /**
+ * Reduces a crawled metadata value to one archive path segment.
+ * Letters, digits, space, comma, colon, plus, and hyphen are kept so a
+ * normal HTTP date is unchanged. Every other character, including dots,
+ * slashes, and backslashes, becomes an underscore. A null value stays
+ * the string "null", which is the filename a missing value produces today.
+ *
+ * @param value crawled metadata value, or null
+ * @return a single safe path segment
+ */
+ static String sanitizeArchiveFileName(String value) {
+ if (value == null) {
+ return "null";
+ }
+ return value.replaceAll("[^A-Za-z0-9 ,:+-]", "_");
+ }
+
/**
* Gets the current date if the given timestamp is empty or null.
*
diff --git a/src/test/org/apache/nutch/tools/TestCommonCrawlDataDumper.java
b/src/test/org/apache/nutch/tools/TestCommonCrawlDataDumper.java
index 575f26b22..e8b53b52f 100644
--- a/src/test/org/apache/nutch/tools/TestCommonCrawlDataDumper.java
+++ b/src/test/org/apache/nutch/tools/TestCommonCrawlDataDumper.java
@@ -17,15 +17,30 @@
package org.apache.nutch.tools;
import java.io.File;
+import java.io.FileInputStream;
import java.lang.reflect.Field;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
+import java.util.ArrayList;
import java.util.Collection;
+import java.util.List;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.FileFilterUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
+import org.apache.nutch.metadata.Metadata;
import org.apache.nutch.metrics.ErrorTracker;
+import org.apache.nutch.protocol.Content;
+import org.apache.nutch.util.NutchConfiguration;
import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
@@ -127,6 +142,134 @@ public class TestCommonCrawlDataDumper {
verify(mockErrorTracker,
times(1)).recordError(ErrorTracker.ErrorType.OTHER);
}
+ /**
+ * With gzip and epoch filenames both enabled, CommonCrawlDataDumper copies
+ * the crawled metadata key "date" into each tar entry name. That value
+ * comes from a response header a remote server controls, so it must not be
+ * able to change the directory an entry is written under.
+ *
+ * What this test does:
+ * <ol>
+ * <li>Writes one segment containing two documents. The first document has
+ * the date "../../../tmp/evil/x", a parent-directory path. The second has
+ * the normal HTTP date "Thu, 25 Sep 2014 04:16:58 GMT".</li>
+ * <li>Runs {@code dump} with gzip and epoch filenames enabled, which is the
+ * code path that builds entry names from the date.</li>
+ * <li>Reads the entry names back from the resulting .tar.gz. The archive is
+ * never extracted, so the test writes nothing outside its temporary
+ * directories.</li>
+ * <li>For every entry, checks that the name has no ".." segment, is not
+ * absolute, and that its final path segment has no slash, no backslash, and
+ * none of the unsafe date text.</li>
+ * <li>Checks that the normal HTTP date still appears in an entry name. This
+ * stops the sanitizer from rewriting legitimate dates.</li>
+ * </ol>
+ *
+ * Why: a regression here would let a crawled site choose where archive
+ * entries land, and the failure would show up only later, when someone
+ * unpacks the archive. Checking the names directly catches the problem at
+ * the point where Nutch writes them.
+ */
+ @Test
+ public void testEpochGzipEntryNamesIgnoreUnsafeDateMetadata()
+ throws Exception {
+ String benignDate = "Thu, 25 Sep 2014 04:16:58 GMT";
+ String unsafeDate = "../../../tmp/evil/x";
+ File segmentDir = Files.createTempDirectory("epoch-segment").toFile();
+ File outputDir = Files.createTempDirectory("epoch-output").toFile();
+ writeContentSegment(segmentDir,
+ "http://example.test/unsafe.html", unsafeDate,
+ "http://example.test/benign.html", benignDate);
+
+ CommonCrawlDataDumper dumper = new CommonCrawlDataDumper(
+ new CommonCrawlConfig());
+ dumper.dump(outputDir, segmentDir, null, true, null, true, "", false);
+
+ List<String> entryNames = readTarEntryNames(outputDir);
+ assertFalse(entryNames.isEmpty(), "gzip epoch dump wrote no tar entries");
+ boolean sawBenign = false;
+ for (String name : entryNames) {
+ assertFalse(name.contains(".."), "entry name contains ..: " + name);
+ assertFalse(name.startsWith("/"), "entry name is absolute: " + name);
+ String segment = name.substring(name.lastIndexOf('/') + 1);
+ assertFalse(segment.contains(".."), segment);
+ assertFalse(segment.indexOf('/') >= 0, segment);
+ assertFalse(segment.indexOf('\\') >= 0, segment);
+ assertFalse(segment.contains(unsafeDate), segment);
+ if (name.contains(benignDate)) {
+ sawBenign = true;
+ }
+ }
+ assertTrue(sawBenign, "normal HTTP date was not kept in an entry name");
+ }
+
+ /**
+ * Writes the two test documents as a single SequenceFile at
+ * {@code segment/content/part-00000/data}. That is the layout the dumper
+ * scans for, so the file must sit in exactly this location. Each document
+ * is stored with its own URL and a metadata record holding only the
+ * lowercase "date" key.
+ */
+ private void writeContentSegment(File segmentDir, String unsafeUrl,
+ String unsafeDate, String benignUrl, String benignDate) throws Exception
{
+ File partDir = new File(segmentDir, "content/part-00000");
+ assertTrue(partDir.mkdirs(), "could not create " + partDir);
+ Configuration conf = NutchConfiguration.create();
+ Path data = new Path(new File(partDir, "data").getAbsolutePath());
+ byte[] body = "<html><body>example</body></html>"
+ .getBytes(StandardCharsets.UTF_8);
+ try (SequenceFile.Writer writer = SequenceFile.createWriter(conf,
+ SequenceFile.Writer.file(data),
+ SequenceFile.Writer.keyClass(Text.class),
+ SequenceFile.Writer.valueClass(Content.class))) {
+ writer.append(new Text(unsafeUrl),
+ contentWithDate(unsafeUrl, unsafeDate, body, conf));
+ writer.append(new Text(benignUrl),
+ contentWithDate(benignUrl, benignDate, body, conf));
+ }
+ }
+
+ /**
+ * Builds a Content record whose metadata has only the lowercase key "date".
+ * The dumper reads that exact key when it builds the entry name, so using a
+ * different case would bypass the code path under test.
+ */
+ private Content contentWithDate(String url, String date, byte[] body,
+ Configuration conf) {
+ Metadata metadata = new Metadata();
+ metadata.set("date", date);
+ return new Content(url, url, body, "text/html", metadata, conf);
+ }
+
+ /**
+ * Finds the single .tar.gz that the dump wrote into {@code outputDir} and
+ * returns every entry name it contains. Fails the test if no archive was
+ * written. Entry names are read as text only; nothing is extracted.
+ */
+ private List<String> readTarEntryNames(File outputDir) throws Exception {
+ File archive = null;
+ File[] files = outputDir.listFiles();
+ if (files != null) {
+ for (File file : files) {
+ if (file.getName().endsWith(".tar.gz")) {
+ archive = file;
+ break;
+ }
+ }
+ }
+ assertTrue(archive != null && archive.isFile(),
+ "gzip epoch dump did not write a tar.gz");
+ List<String> names = new ArrayList<>();
+ try (TarArchiveInputStream tar = new TarArchiveInputStream(
+ new GzipCompressorInputStream(new FileInputStream(archive)))) {
+ TarArchiveEntry entry;
+ while ((entry = tar.getNextEntry()) != null) {
+ names.add(entry.getName());
+ }
+ }
+ return names;
+ }
+
private boolean hasFile(String fileName, Collection<File> files) {
for (File f : files) {
if (f.getName().equals(fileName)) {