rzo1 commented on PR #2207:
URL: https://github.com/apache/stormcrawler/pull/2207#issuecomment-5895712101
Here are tests for (a), (b) and (c) as a patch. On this branch
`testQueryNormalizedLikePath` and `testAnchorKeptWhenURLRebuilt` fail,
`testHostWithUnderscoreWhenURLRebuilt` passes.
<details>
<summary>Patch for BasicURLNormalizerTest</summary>
```diff
diff --git
a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java
b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java
index 922f1b0b..f4a27bcf 100644
---
a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java
+++
b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java
@@ -365,6 +365,53 @@ class BasicURLNormalizerTest {
"http://A.example/r?u=https%3A%2F%2Fb.example&t=~!q&utm_source=x";));
}
+ @Test
+ void testQueryNormalizedLikePath() throws MalformedURLException {
+ URLFilter urlFilter = createFilter(new
ObjectNode(JsonNodeFactory.instance));
+ URL testSourceUrl = URLUtil.toURL("http://a.example/";);
+ // non-ASCII is escaped in the path, so it should be in the query
too
+ assertEquals(
+ "http://a.example/%C3%A9?q=%C3%A9";,
+ urlFilter.filter(
+ testSourceUrl, new Metadata(),
"http://a.example/\u00e9?q=\u00e9";));
+ // hex digits of an escape are uppercased
+ assertEquals(
+ "http://a.example/p?q=%E2%84%A2";,
+ urlFilter.filter(testSourceUrl, new Metadata(),
"http://a.example/p?q=%e2%84%a2";));
+ // escaped unreserved characters are decoded
+ assertEquals(
+ "http://a.example/p?u=~user";,
+ urlFilter.filter(testSourceUrl, new Metadata(),
"http://a.example/p?u=%7Euser";));
+ }
+
+ @Test
+ void testHostWithUnderscoreWhenURLRebuilt() throws
MalformedURLException {
+ URLFilter urlFilter = createFilter(new
ObjectNode(JsonNodeFactory.instance));
+ URL testSourceUrl = URLUtil.toURL("http://a.example/";);
+ // not rebuilt: passes through
+ assertEquals(
+ "http://a_b.example/p";,
+ urlFilter.filter(testSourceUrl, new Metadata(),
"http://a_b.example/p";));
+ // rebuilt because of the uppercase host: same result
+ assertEquals(
+ "http://a_b.example/p";,
+ urlFilter.filter(testSourceUrl, new Metadata(),
"http://A_b.example/p";));
+ }
+
+ @Test
+ void testAnchorKeptWhenURLRebuilt() throws MalformedURLException {
+ URLFilter urlFilter = createFilter(false, false);
+ URL testSourceUrl = URLUtil.toURL("http://a.example/";);
+ // not rebuilt: the anchor is kept
+ assertEquals(
+ "http://a.example/p?x=1#sec";,
+ urlFilter.filter(testSourceUrl, new Metadata(),
"http://a.example/p?x=1#sec";));
+ // rebuilt because of the uppercase host: the anchor should be kept
too
+ assertEquals(
+ "http://a.example/p?x=1#sec";,
+ urlFilter.filter(testSourceUrl, new Metadata(),
"http://A.example/p?x=1#sec";));
+ }
+
private JsonNode getArrayNode(List<String> queryElementsToRemove) {
ObjectMapper mapper = new ObjectMapper();
return mapper.valueToTree(queryElementsToRemove);
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
