This is an automated email from the ASF dual-hosted git repository.
dependabot[bot] pushed a change to branch
dependabot/maven/git-code-format-maven-plugin.version-6.0
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git
omit 6b1b5f3c Bump git-code-format-maven-plugin.version from 5.4 to 6.0
add c55d3200 Bump langchain4j.version from 1.11.0 to 1.12.2
add 6fcd0897 Bump org.apache.maven.plugins:maven-resources-plugin from
3.4.0 to 3.5.0
add be6631b4 Bump storm-client.version from 2.8.3 to 2.8.4 (#1823)
add 5f97d7f8 Fix checkstyle issues (#1816)
add dd1e2e85 Bump org.mockito:mockito-core from 5.22.0 to 5.23.0 (#1830)
add 3246c2a1 Bump opensearch.version from 2.19.4 to 2.19.5 (#1829)
add 72cbfbf2 Minor: Regenerated License File for
3246c2a1de68f68e6af0b4701bb05e14d9a2328b (#1828)
add bb33f929 fix(core): make Metadata keys consistently case-insensitive
(#1827)
add 50c30074 Bump actions/cache from 5.0.3 to 5.0.4 (#1832)
add b8c18c08 Next Storm (2.8.5) removed its dependency towards lang 2.x -
we shouldn'T use the shaded dependency in SC anyway. (#1835)
add ebec4bec Update jackson to be in-line with the one in Storm (#1836)
add e001d4c8 Minor: Regenerated License File for
ebec4beccef2a215d21b5b5b1dfb337826007f65 (#1837)
add 01e25b7e Bump com.ibm.icu:icu4j from 78.2 to 78.3 (#1838)
add bcc73482 #1597 - Java 20 deprecates all constructors of the class
java.net.URL (#1834)
add da29b591 Bump org.apache.rat:apache-rat-plugin from 0.17 to 0.18
(#1839)
add d6c07fd1 Minor: Regenerated License File for
bcc7348269da6999866d0e7ce6102b729cf36312 (#1840)
add fbefb88c Improve documentation (missing configs, etc.) (#1843)
add 566db1fb WARCSpout: use jwarc utility method to decode WARC payload
(#1845)
add 3af0dde8 docs: add PowerShell scripts (#1851)
add 1dea53b5 Upgrade to Apache Storm 2.8.5 (#1847)
add e3062a18 Replace Thread.sleep polling with Awaitility in tests (#1852)
add 52bd4017 Bump tika.version from 3.2.3 to 3.3.0 (#1850)
add 1d422b5e Minor: Regenerated License File for
52bd4017863fd3f49f77511e0be5650983544d54 (#1853)
add 55d744f7 remove maximumsize for cache in abstractupdaterBolt (#1854)
add a08a456b Fix race condition while having different proxies in
different threads #1247 (#1855)
add 25684d47 Replace XSoup with JSoup built-in XPath support (#1856)
add 753d68df Minor: Regenerated License File for
25684d47033844cabc3dbeab9b56ef7af0b06b04 (#1858)
add 7f10122d Add XPath JSoup filter tests for XSoup backward compatibility
(#1857)
add df7dfe3c Add lowercaseElementNames unit test and make method public
(#1860)
add c179baef Add bolt-level timeout for fetcher threads (#1861)
add d485783c Fix bugs and resource leaks in Apache HttpClient protocol
implementation (#1863)
add c523032a Fix thread-safety and minor issues in OkHttp protocol (#1862)
add 45a9755b Fix double-close race on static client in OpenSearch
AbstractSpout (#1864)
add 5e5fc3b2 Fix client leak on BulkProcessor or Sniffer construction
failure (#1865)
add 29d3c266 Fix timer and client leak in OpenSearch JSONResourceWrapper
(#1866)
add cdac7e46 Add cleanup() to URLFilter and fix timer/client leak in
JSONURLFilterWrapper (#1867)
add 47d6bab0 Fix stale log messages, missing log args, and inconsistent
metric scopes (#1871)
add b4e97ec4 #585 Add Storm V2 metrics support with backward-compatible
bridge (#1846)
add 197f8879 Extract DelegateRefresher to deduplicate JSON resource
wrapper logic (#1870)
add 55a0ff2e Fix Solr 10 migration (#1833)
add 1548d2b7 Minor: Regenerated License File for
55a0ff2e5fa13a542849c73d08665d5f4502a0ec (#1872)
add 2d306c4e Extract WaitAckCache to deduplicate OpenSearch bulk response
handling (#1869)
add 92af9e6a Bump selenium.version from 4.41.0 to 4.43.0 (#1873)
add d74d2c83 Bump com.microsoft.playwright:playwright from 1.58.0 to
1.59.0 (#1875)
add 70645c2e Bump org.netpreserve:jwarc from 0.35.0 to 0.36.0 (#1876)
add 9d3ff571 Bump langchain4j.version from 1.12.2 to 1.13.0 (#1877)
add 659dda56 Bump storm-client.version from 2.8.5 to 2.8.6 (#1874)
add eda73fd0 Bump peter-evans/create-pull-request from 8.1.0 to 8.1.1
(#1878)
add e5dac9fa Minor: Regenerated License File for
659dda56ce6fb7c72a4b1105c6b2da9582ce9173 (#1879)
add cfa6632a Bump actions/cache from 5.0.4 to 5.0.5 (#1881)
add fecc9517 Bump com.google.guava:guava from 33.5.0-jre to 33.6.0-jre
(#1882)
add 5c229d86 Add opensearch-java module using java client (#1868)
add dc5e809b Minor: Regenerated License File for
5c229d865b0f315f8bfe4c095f6483032b0fd6e5 (#1884)
add 0394f83b feat(playwright): capture rendered DOM on non-2xx responses
(#1891)
add a4c7af78 Bump org.jsoup:jsoup from 1.22.1 to 1.22.2 (#1885)
add 7c5e8e4d Bump langchain4j.version from 1.13.0 to 1.13.1 (#1888)
add 6b55c4c3 Bump commons-codec:commons-codec from 1.21.0 to 1.22.0 (#1886)
add 5e8c1440 Bump storm-client.version from 2.8.6 to 2.8.7 (#1887)
add a69f5649 Minor: Regenerated License File for
5e8c14402865dbb15f089ce4da9a97af37209cec (#1892)
add 6b464c4d fix: my-sql-connector migration flaky test (#1893)
add 2c6d91d9 Bump com.github.ben-manes.caffeine:caffeine from 3.2.3 to
3.2.4 (#1894)
add 8c20068d Bump langchain4j.version from 1.13.1 to 1.14.0 (#1895)
add cd97eb8c Minor: Regenerated License File for
8c20068d9da634ef7a91eabc3754a5810a27ff42 (#1896)
add 7a17307a OkHttp protocol fails to cancel request after (#1900)
add 3d55b2bd feat(playwright): add JsRenderingDetector parse filter & bolt
(#1898)
add 65468216 Bump org.apache:apache from 37 to 38 (#1902)
add a9c4fec9 Bump langchain4j.version from 1.14.0 to 1.14.1 (#1903)
add 10ab32d6 feat(playwright): add PageAction extension point with
built-in actions (#1897)
add e852b0d4 Minor: Regenerated License File for
10ab32d62eaf78975bb374c1e83aa8aec2a7239a (#1905)
add 59cb3fda Fix Playwright tests failing when run together (#1906)
add fc1efb71 Increase checkstyle severities (#1907)
add 54f46360 Bump langchain4j.version from 1.14.1 to 1.15.0 (#1908)
add ed7572d0 Bump selenium.version from 4.43.0 to 4.44.0 (#1909)
add b45f8d7e Bump org.slf4j:slf4j-simple from 2.0.17 to 2.0.18 (#1910)
add e6648c12 Minor: Regenerated License File for
54f463608f448611a826d4a1538b6cea86ec80b1 (#1911)
add af89f3f4 Minor: Regenerated License File for
e6648c12d979121abdd3f9bd0a4fde33887419a3 (#1913)
add 1a6f29c7 Upgrade to Storm 2.8.8 (#1912)
add 4a76a83d Minor: Regenerated License File for
1a6f29c745d83cb371461e1900c94c15f072d8b4 (#1914)
add bf7252fe Bump junit.version from 6.0.3 to 6.1.0 (#1915)
add c4edff04 Bump com.microsoft.playwright:playwright from 1.59.0 to
1.60.0 (#1916)
add fa5ba6a0 Bump org.apache.maven.plugins:maven-enforcer-plugin from
3.6.2 to 3.6.3 (#1917)
add a096fe3c Minor: Regenerated License File for
fa5ba6a07dda51f6d98e0f59a0b080899e749df0 (#1918)
add 6c20eda6 Minor cleanup for shaded artifacts
add 08c120d2 [maven-release-plugin] prepare release stormcrawler-3.6.0
add 1fb26cb1 [maven-release-plugin] prepare for next development iteration
add bfb66d87 Bump langchain4j.version from 1.15.0 to 1.15.1
add 5cc91602 Bump tika.version from 3.3.0 to 3.3.1
add 1aa443a9 Bump org.apache.maven.plugins:maven-surefire-plugin from
3.5.5 to 3.5.6
add 9e6880f7 Minor: Regenerated License File for
1aa443a970bc465e685ceeb90a44c6f0d57e7294
add 5ecbfc75 Bump actions/checkout from 6.0.2 to 6.0.3
add 46bf3f26 Fix missing space between block and inline sibling elements
in JSoupTextExtractor (#1926)
add 14eca05d Add AGENTS.md + SECURITY.md pointing to the security model
(scanner discoverability)
add 6a3785f9 fix: handle NumberFormatException in FetcherBolt and minor
cleanup (#1931)
add 9c62f2ca fix: replace non-thread-safe SimpleDateFormat and shared
Matcher instances (#1929)
add 3a4d99a2 fix: close streams and stop tracing in finally to prevent
resource leaks (#1930)
add a84e660a Bump org.jacoco:jacoco-maven-plugin from 0.8.14 to 0.8.15
add d33cd0e3 Bump langchain4j.version from 1.15.1 to 1.16.1
add c93a1656 Minor: Regenerated License File for
d33cd0e3e16d4310cb37f245a73ea0bcb9c979c1
add aab3e18b 1143 Add per-URL proxy metadata selection (#1935)
add 323de615 docs: fix StormCrawler documentation links (#1940)
add a2769591 #1938 - Migrate AWS Java SDK from 1.x to 2.x (#1938) (#1939)
add df9b05d0 Minor: Regenerated License File for
a2769591f9393ff808266def83cffe427ee7e538 (#1941)
add 37f2786f Bump software.amazon.awssdk:bom from 2.46.7 to 2.46.10 (#1947)
add d232443c Bump langchain4j.version from 1.16.1 to 1.16.2 (#1945)
add f324adf0 Bump okhttp.version from 5.3.2 to 5.4.0 (#1946)
add 3b786442 Minor: Regenerated License File for
37f2786fc0df6fc3159ebca6565e4c90c5e800ff (#1949)
add 2456ce39 Bump org.opensearch.client:opensearch-java from 3.8.0 to
3.9.0 (#1948)
add 767491f0 Minor: Regenerated License File for
2456ce391e9e0e5f6ac6baae3dedf28e877f8297 (#1950)
add 62177234 FileSpout: add support for gzip and bz2 files (#1952)
add 82d7f752 #1107 - Add okhttp IP address filter (#1942)
add f77eb24f Bump actions/setup-java from 5.2.0 to 5.3.0 (#1957)
add 6ecd0e0f #1951 refactor: replace linear proxy scan with lookup map in
MultiProxyManager (#1954)
add aeb16a63 #1955 - Make FileSpout work in distributed mode (#1956)
add b927a04c build(deps): bump actions/checkout from 6.0.3 to 7.0.0 (#1958)
add f4f5b602 build(deps): bump langchain4j.version from 1.16.2 to 1.16.3
(#1959)
add 2f1d91be build(deps): bump software.amazon.awssdk:bom from 2.46.10 to
2.46.15 (#1961)
add 0786768d build(deps): bump selenium.version from 4.44.0 to 4.45.0
(#1960)
add 5f876a5c #466 - Handle text/plain content in JSoupParserBolt (#1943)
add 7f32057a Minor: Regenerated License File for
5f876a5cfcbecfb193539ea1bf8cc7fae37abc4a (#1962)
add 74dd430b use placeholder instead of older version of archetypes in
README + update link (#1963)
add 830b40bb build(deps): bump actions/cache from 5.0.5 to 6.0.0 (#1964)
add e1c3edb3 build(deps): bump actions/setup-java from 5.3.0 to 5.4.0
(#1965)
add 8b15ea85 build(deps): bump software.amazon.awssdk:bom from 2.46.15 to
2.46.17 (#1967)
add 3f6aae7a build(deps): bump junit.version from 6.1.0 to 6.1.1 (#1966)
add a51348dc build(deps): bump actions/cache from 6.0.0 to 6.1.0 (#1970)
add 75573f86 Minor: Regenerated License File for
a51348dca7f497f205dbe7580ff07258a9018dd7 (#1971)
add 2723168a build(deps): bump org.apache:apache from 38 to 39 (#1969)
add 0b852c8c build(deps): bump langchain4j.version from 1.16.3 to 1.17.0
(#1968)
add cae1f5b8 Minor: Regenerated License File for
0b852c8cff7f2c7a9e72b8b3498092871ca6ff9f (#1972)
add ebda1771 Queue stream (#1974)
add 0189bd2a Block the URLFrontier queue on Retry-After via the queue
stream (#1973)
add 3c65dd4d build(deps): bump software.amazon.awssdk:bom from 2.46.17 to
2.46.21 (#1978)
add a4a99dc2 build(deps): bump com.microsoft.playwright:playwright (#1977)
add 491859c3 build(deps): bump opensearch.version from 2.19.5 to 2.19.6
(#1976)
add 74e02717 build(deps): bump langchain4j.version from 1.17.0 to 1.17.1
(#1975)
add e2361403 Minor: Regenerated License File for
74e027177364a49c490183ed38bfb8bdc433bc38 (#1980)
add b60589b2 Port QueueBolt (queue stream) to the opensearch-java module
(#1981)
add 63be7ae4 build(deps): bump actions/setup-java from 5.4.0 to 5.5.0
(#1983)
add 9ba343a7 Adding Green Web Foundation to Powered By (#1985)
add 4a41fa96 Don't trigger Mvn workflow for changes to the docs (#1986)
add 65f6a12e Pluggable robot rule parser (#1987)
add 7b5bf6ce Adaptive back-off for rate-limited hosts via the queue stream
(#1984)
add af133c93 Workaround to activate LogViewer in docker compose (#1992)
add 007b994a Minor: Regenerated License File for
af133c93d44148536bae75593e4046da05347688 (#1993)
add 137627ca Verbatim headers for Playwright (#1996)
add 21577865 build(deps): bump langchain4j.version from 1.17.1 to 1.17.2
(#1988)
add 2280461f build(deps): bump junit.version from 6.1.1 to 6.1.2 (#1989)
add 8917113d build(deps): bump selenium.version from 4.45.0 to 4.46.0
(#1990)
add d4e7373f build(deps): bump software.amazon.awssdk:bom from 2.46.21 to
2.47.5 (#1991)
add 7b92388e Minor: Regenerated License File for
d4e7373f2b50d987f5f4ffe806f5d8641ae57db5 (#1999)
add 86647d34 [WARC] Format WARC-IP-Address header values following the
WARC community recommendations (#1997)
add eaa6869e #2001 Document adaptive scheduler metadata (#2002)
add 3664644a bugfix - max content not applied for robots.txt when
redirecting (#2004)
add 0e01b67e build(deps): bump actions/setup-java from 5.5.0 to 5.6.0
(#2003)
add 18dac8c5 build(deps): bump software.amazon.awssdk:bom from 2.47.5 to
2.48.3 (#2008)
add 04871dab build(deps): bump langchain4j.version from 1.17.2 to 1.18.0
(#2007)
add 83f90633 Minor: Regenerated License File for
04871dab3d97385c13768cc9f40914589fbbb92a (#2009)
add 17c14a86 build(deps): bump actions/checkout from 7.0.0 to 7.0.1 (#2010)
add 548726f4 Fix #304: Use HTML test document instead of sitemap XML in
SubDocumentsFilterTest (#2005)
add cc511f9f Resolve relative outlinks against <base href> in
JSoupParserBolt (#2011)
add e87846ac Mark Selenium module as deprecated (#2019)
add 5eb99d0a Mark CloudSearch as deprecated. Fixes #2016 (#2018)
add 54c4a6d3 Allow indexer bolts to use a metadata value as the document
ID (#2012)
add 3c06a126 Mark HTTPClient as deprecated (#2020)
add 8667217f build(deps): bump org.apache.maven.plugins:maven-jar-plugin
(#2025)
add ee983434 build(deps): bump tika.version from 3.3.1 to 3.3.2 (#2023)
add f1e35b21 build(deps): bump software.amazon.awssdk:bom from 2.48.3 to
2.49.3 (#2022)
add cd5f59c5 Minor: Regenerated License File for
ee983434cdd00cfbf44d0c6b3d79f124942bf682 (#2028)
add b13ed812 Minor: Regenerated License File for
f1e35b21261fc43b6990bae7f633c071b7947d57 (#2029)
add 725781f1 Minor: Regenerated License File for
cd5f59c51bdae43202c2c0f408a98fe01b99ae83 (#2030)
add d193e6cd #2026 - http.agent.name is dropped from the robots.txt match
set when http.robots.agents is set (#2027)
add dc1f70bb fix: emit anchored tuples #2021 (#2031)
add 9304adac Enforce robots.txt Crawl-delay in URLFrontier queues (#2000)
add 6e690830 Upgrade to Storm 2.8.9 (#2032)
add e3297e48 Regenerated License file after dependency upgrades (#2033)
add 0dc2ab5e [maven-release-plugin] prepare release stormcrawler-3.7.0
add 883b8546 [maven-release-plugin] prepare for next development iteration
add 214b23e1 Fix typo in OpenSearch configuration comment
add 747f5d0a Fix broken link to Guava's CacheBuilderSpec
add 8f94a75f Fix broken anchor in Asciidoc
add 2dddb1da build(deps): bump software.amazon.awssdk:bom from 2.49.3 to
2.50.0
add 306474da build(deps): bump actions/setup-java from 5.6.0 to 5.7.0
add caec3bcf build(deps): bump commons-codec:commons-codec from 1.22.0 to
1.22.1
add de6977df build(deps): bump langchain4j.version from 1.18.0 to 1.18.1
add 9b72b5a5 build(deps): bump com.mysql:mysql-connector-j from 9.7.0 to
26.7.0
add e4d30f71 build(deps): bump org.jsoup:jsoup from 1.22.2 to 1.23.1
add 738c3024 Upgrade to Apache Storm 3.0.0 and Java 25 (#2017)
add 00b5be02 Minor: Regenerated License File for
738c3024793d609bbb0a5b742c6ec5f50566038b (#2044)
add 54fcf5b5 Upgrade to Storm 3 (#2045)
add e2fccebf Minor: Regenerated License File for
54fcf5b58d6e070b2d0fe89df130b66eafbf0449 (#2046)
add e1d54a65 refresh NOTICE copyright year to 2026 (#2047)
add bbdebac9 build(deps): bump com.microsoft.playwright:playwright from
1.61.0 to 1.62.0 (#2048)
add e6c4f7b2 build(deps): bump software.amazon.awssdk:bom from 2.50.0 to
2.51.2 (#2049)
add a0838c86 Regenerated License file after dependency upgrades (#2051)
add 5128abb3 build(deps): bump git-code-format-maven-plugin.version from
5.4 to 6.0
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (6b1b5f3c)
\
N -- N -- N
refs/heads/dependabot/maven/git-code-format-maven-plugin.version-6.0 (5128abb3)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.github/workflows/main.yml | 12 +-
.github/workflows/maven.yml | 20 +-
.github/workflows/snapshots.yaml | 6 +-
AGENTS.md | 7 +
NOTICE | 2 +-
README.md | 11 +-
SECURITY.md | 10 +
THIRD-PARTY.txt | 428 +++++++++--------
archetype/pom.xml | 4 +-
.../main/resources/archetype-resources/README.md | 2 +-
.../archetype-resources/docker-compose.yml | 12 +-
.../src/main/resources/archetype-resources/pom.xml | 6 +-
checkstyle.xml | 37 +-
core/pom.xml | 34 +-
.../apache/stormcrawler/ConfigurableTopology.java | 6 +-
.../java/org/apache/stormcrawler/Constants.java | 9 +
.../java/org/apache/stormcrawler/Metadata.java | 67 ++-
.../apache/stormcrawler/bolt/FeedParserBolt.java | 4 +-
.../org/apache/stormcrawler/bolt/FetcherBolt.java | 170 +++++--
.../apache/stormcrawler/bolt/JSoupParserBolt.java | 204 +++++---
.../stormcrawler/bolt/SimpleFetcherBolt.java | 124 +++--
.../stormcrawler/bolt/SiteMapParserBolt.java | 18 +-
.../stormcrawler/bolt/StatusEmitterBolt.java | 7 +
.../apache/stormcrawler/bolt/URLFilterBolt.java | 11 +-
.../stormcrawler/bolt/URLPartitionerBolt.java | 11 +-
.../apache/stormcrawler/filtering/URLFilter.java | 9 +
.../apache/stormcrawler/filtering/URLFilters.java | 42 +-
.../filtering/basic/BasicURLNormalizer.java | 30 +-
.../stormcrawler/filtering/host/HostURLFilter.java | 3 +-
.../filtering/regex/FastURLFilter.java | 287 +++++------
.../filtering/regex/RegexURLFilterBase.java | 34 +-
.../filtering/robots/RobotsFilter.java | 3 +-
.../stormcrawler/indexing/AbstractIndexerBolt.java | 53 ++-
.../apache/stormcrawler/jsoup/LinkParseFilter.java | 5 +-
.../org/apache/stormcrawler/jsoup/XPathFilter.java | 122 ++++-
.../stormcrawler/metrics/CrawlerMetrics.java | 315 ++++++++++++
.../stormcrawler/metrics/DualCounterMetric.java} | 33 +-
.../stormcrawler/metrics/DualReducedMetric.java | 40 +-
.../Status.java => metrics/ScopedCounter.java} | 36 +-
.../ScopedReducedMetric.java} | 22 +-
.../stormcrawler/metrics/v1/V1CounterMetric.java} | 26 +-
.../stormcrawler/metrics/v1/V1ReducedMetric.java} | 26 +-
.../stormcrawler/metrics/v2/V2CounterMetric.java | 50 ++
.../metrics/v2/V2HistogramReducedMetric.java | 60 +++
.../metrics/v2/V2MeterReducedMetric.java | 59 +++
.../apache/stormcrawler/parse/JSoupFilters.java | 28 +-
.../stormcrawler/parse/JSoupTextExtractor.java | 4 +-
.../apache/stormcrawler/parse/ParseFilters.java | 26 +-
.../parse/filter/CollectionTagger.java | 108 ++---
.../parse/filter/DebugParseFilter.java | 19 +-
.../stormcrawler/parse/filter/LinkParseFilter.java | 5 +-
.../parse/filter/MD5SignatureParseFilter.java | 2 +-
.../parse/filter/MimeTypeNormalization.java | 2 +-
.../stormcrawler/parse/filter/XPathFilter.java | 3 +
.../persistence/AbstractQueryingSpout.java | 26 +-
.../persistence/AbstractStatusUpdaterBolt.java | 43 +-
.../protocol/AbstractHttpProtocol.java | 12 +-
.../org/apache/stormcrawler/protocol/CIDR.java | 88 ++++
.../stormcrawler/protocol/DelegatorProtocol.java | 4 +-
.../protocol/HttpRobotRulesParser.java | 7 +-
.../stormcrawler/protocol/IPFilterRules.java | 137 ++++++
.../stormcrawler/protocol/ProtocolFactory.java | 6 +-
.../apache/stormcrawler/protocol/RobotRules.java | 11 +
.../stormcrawler/protocol/RobotRulesParser.java | 18 +-
.../stormcrawler/protocol/file/FileResponse.java | 16 +-
.../protocol/httpclient/HttpProtocol.java | 116 +++--
.../stormcrawler/protocol/okhttp/HttpProtocol.java | 108 ++++-
.../stormcrawler/proxy/MultiProxyManager.java | 31 +-
.../apache/stormcrawler/proxy/ProxyMetadata.java | 148 ++++++
.../proxy/{ProxyManager.java => ProxyUtils.java} | 28 +-
.../org/apache/stormcrawler/proxy/SCProxy.java | 55 ++-
.../stormcrawler/proxy/SingleProxyManager.java | 44 +-
.../org/apache/stormcrawler/spout/FileSpout.java | 112 +++--
.../org/apache/stormcrawler/spout/MemorySpout.java | 59 +--
.../apache/stormcrawler/util/CookieConverter.java | 26 +-
.../apache/stormcrawler/util/MetadataTransfer.java | 8 +-
.../apache/stormcrawler/util/PerSecondReducer.java | 10 +-
.../org/apache/stormcrawler/util/RefreshTag.java | 9 +-
.../apache/stormcrawler/util/RetryAfterParser.java | 101 ++++
.../apache/stormcrawler/util/URLPartitioner.java | 4 +-
.../stormcrawler/util/URLStreamGrouping.java | 2 +-
.../java/org/apache/stormcrawler/util/URLUtil.java | 108 ++++-
core/src/main/resources/crawler-default.yaml | 46 +-
.../java/org/apache/stormcrawler/MetadataTest.java | 42 ++
.../stormcrawler/TestMetadataSerialization.java | 1 +
.../apache/stormcrawler/TestOutputCollector.java | 23 +-
.../java/org/apache/stormcrawler/TestUtil.java | 48 +-
.../stormcrawler/bolt/AbstractFetcherBoltTest.java | 159 ++++++-
.../stormcrawler/bolt/FeedParserBoltTest.java | 1 +
.../apache/stormcrawler/bolt/FetcherBoltTest.java | 114 +++++
.../stormcrawler/bolt/JSoupParserBoltTest.java | 110 ++++-
.../stormcrawler/bolt/SimpleFetcherBoltTest.java | 107 +++++
.../stormcrawler/bolt/SiteMapParserBoltTest.java | 1 +
.../stormcrawler/bolt/URLFilterBoltTest.java | 134 ++++++
.../stormcrawler/filtering/BasicURLFilterTest.java | 10 +-
.../filtering/BasicURLNormalizerTest.java | 50 +-
.../stormcrawler/filtering/FastURLFilterTest.java | 10 +-
.../stormcrawler/filtering/HostURLFilterTest.java | 10 +-
.../stormcrawler/filtering/MaxDepthFilterTest.java | 10 +-
.../filtering/MetadataFilterFromJsonTest.java | 20 +-
.../stormcrawler/filtering/MetadataFilterTest.java | 30 +-
.../stormcrawler/filtering/RegexFilterTest.java | 16 +-
.../ClassInheritingFromAbstractAndInterface.java | 1 +
.../ClassInheritingFromAbstractClassOnly.java | 1 +
.../ClassInheritingFromOpenClass.java | 1 +
.../ClassWithoutValidConstructor.java | 1 +
.../initialisation/FinalClassToInitialize.java | 1 +
.../helper/initialisation/SimpleOpenClass.java | 1 +
.../helper/initialisation/base/AbstractClass.java | 1 +
.../helper/initialisation/base/ITestInterface.java | 1 +
.../OpenClassWithAbstractClassAndInterface.java | 1 +
.../stormcrawler/indexer/BasicIndexingTest.java | 40 ++
.../apache/stormcrawler/indexer/DummyIndexer.java | 17 +-
.../apache/stormcrawler/indexer/IndexerTester.java | 1 +
.../apache/stormcrawler/json/JsoupFilterTest.java | 1 +
.../stormcrawler/jsoup/JSoupFiltersTest.java | 78 +++
.../stormcrawler/metrics/CrawlerMetricsTest.java | 134 ++++++
.../DualCounterMetricTest.java} | 39 +-
.../DualReducedMetricTest.java} | 38 +-
.../metrics/v2/V2CounterMetricTest.java | 82 ++++
.../metrics/v2/V2HistogramReducedMetricTest.java | 85 ++++
.../metrics/v2/V2MeterReducedMetricTest.java | 78 +++
.../stormcrawler/parse/DuplicateLinksTest.java | 1 +
.../stormcrawler/parse/JSoupTextExtractorTest.java | 30 ++
.../apache/stormcrawler/parse/ParsingTester.java | 1 +
.../stormcrawler/parse/StackOverflowTest.java | 5 +-
.../parse/filter/CSVMetadataFilterTest.java | 1 +
.../parse/filter/CollectionTaggerTest.java | 1 +
.../parse/filter/SubDocumentsFilterTest.java | 11 +-
.../parse/filter/SubDocumentsParseFilter.java | 78 ++-
.../stormcrawler/parse/filter/XPathFilterTest.java | 1 +
.../persistence/AdaptiveSchedulerTest.java | 1 +
.../persistence/DefaultSchedulerTest.java | 43 +-
.../stormcrawler/persistence/URLBufferTest.java | 1 +
.../protocol/AbstractProtocolTest.java | 37 +-
.../org/apache/stormcrawler/protocol/CIDRTest.java | 90 ++++
.../protocol/DelegationProtocolTest.java | 1 +
.../stormcrawler/protocol/DummyProtocol.java | 1 +
.../protocol/HttpRobotRulesParserRedirectTest.java | 1 +
.../protocol/HttpRobotRulesParserTest.java | 1 +
.../stormcrawler/protocol/IPFilterRulesTest.java | 101 ++++
.../protocol/RobotRulesParserAgentNamesTest.java | 94 ++++
.../HttpClientProtocolProxyManagerTest.java | 70 +++
.../okhttp/HttpProtocolProxyConcurrencyTest.java | 132 ++++++
.../stormcrawler/proxy/MultiProxyManagerTest.java | 172 +++++++
.../org/apache/stormcrawler/proxy/SCProxyTest.java | 72 +++
.../stormcrawler/proxy/SingleProxyManagerTest.java | 264 +++++++++++
.../apache/stormcrawler/spout/FileSpoutTest.java | 130 +++++
.../spout/mocks/FileSpoutOutputCollectorMock.java | 1 +
.../spout/mocks/FileSpoutTopologyContextMock.java | 1 +
.../apache/stormcrawler/util/ConfUtilsTest.java | 1 +
.../stormcrawler/util/CookieConverterTest.java | 3 +-
.../stormcrawler/util/InitialisationUtilTest.java | 7 +-
.../stormcrawler/util/MetadataTransferTest.java | 7 +-
.../apache/stormcrawler/util/RefreshTagTest.java | 1 +
.../stormcrawler/util/RetryAfterParserTest.java | 167 +++++++
.../apache/stormcrawler/util/RobotsTagsTest.java | 1 +
.../stormcrawler/util/URLUtilSanitizeTest.java | 98 ++++
.../apache/stormcrawler/util/URLUtilToURLTest.java | 112 +++++
.../{duplicateLinks.html => subdocuments.html} | 12 +-
core/src/test/resources/test.urlfilterbolt.json | 11 +
.../test/resources/test.xpath.jsoupfilters.json | 22 +
.../{javascriptLinks.html => xpath-test.html} | 25 +-
docs/pom.xml | 2 +-
docs/src/main/asciidoc/architecture.adoc | 6 +-
docs/src/main/asciidoc/configuration.adoc | 527 +++++++++++++++++++--
docs/src/main/asciidoc/extending.adoc | 421 ++++++++++++++++
docs/src/main/asciidoc/index.adoc | 2 +
docs/src/main/asciidoc/internals.adoc | 127 ++++-
docs/src/main/asciidoc/powered-by.adoc | 3 +-
docs/src/main/asciidoc/quick-start.adoc | 59 ++-
external/ai/pom.xml | 4 +-
.../stormcrawler/ai/AbstractLLMTextExtractor.java | 1 +
.../stormcrawler/ai/OpenAITextExtractor.java | 1 +
.../ai/listener/LlmResponseListener.java | 1 +
.../stormcrawler/ai/listener/NoOpListener.java | 1 +
.../stormcrawler/ai/OpenAITextExtractorTest.java | 1 +
external/aws/README.md | 19 +-
external/aws/aws-conf.yaml | 8 +
external/aws/pom.xml | 57 ++-
.../aws/bolt/CloudSearchConstants.java | 6 +
.../aws/bolt/CloudSearchIndexerBolt.java | 155 +++---
.../stormcrawler/aws/bolt/CloudSearchUtils.java | 32 +-
.../stormcrawler/aws/s3/AbstractS3CacheBolt.java | 57 ++-
.../apache/stormcrawler/aws/s3/S3CacheChecker.java | 55 +--
.../org/apache/stormcrawler/aws/s3/S3Cacher.java | 49 +-
.../stormcrawler/aws/s3/S3ContentCacher.java | 3 +-
.../stormcrawler/aws/s3/S3CacheBoltTest.java | 168 +++++++
external/langid/pom.xml | 2 +-
.../stormcrawler/parse/filter/LanguageID.java | 1 +
external/opensearch-java/README.md | 74 +++
external/{opensearch => opensearch-java}/pom.xml | 33 +-
.../opensearch/AsyncBulkProcessor.java | 300 ++++++++++++
.../opensearch/BulkItemResponseToFailedFlag.java | 71 +++
.../apache/stormcrawler/opensearch/Constants.java | 1 +
.../stormcrawler/opensearch/DelegateRefresher.java | 181 +++++++
.../stormcrawler/opensearch/IndexCreation.java | 85 ++--
.../opensearch/OpenSearchConnection.java | 449 ++++++++++++++++++
.../stormcrawler/opensearch/WaitAckCache.java | 340 +++++++++++++
.../stormcrawler/opensearch/bolt/DeletionBolt.java | 180 +++++++
.../stormcrawler/opensearch/bolt/IndexerBolt.java | 297 ++++++++++++
.../opensearch/filtering/JSONURLFilterWrapper.java | 83 ++++
.../opensearch/metrics/MetricsConsumer.java | 42 +-
.../opensearch/metrics/MetricsReporter.java | 219 +++++++++
.../opensearch/metrics/StatusMetricsBolt.java | 185 ++++++++
.../parse/filter/JSONResourceWrapper.java | 79 +++
.../opensearch/persistence/AbstractSpout.java | 125 ++++-
.../opensearch/persistence/AggregationSpout.java | 427 +++++++++++++++++
.../opensearch/persistence/HybridSpout.java | 263 ++++++++++
.../opensearch/persistence/QueueBolt.java | 191 ++++++++
.../opensearch/persistence/StatusUpdaterBolt.java | 358 ++++++++++++++
.../opensearch/AsyncBulkProcessorTest.java | 241 ++++++++++
.../opensearch/DelegateRefresherTest.java | 256 ++++++++++
.../opensearch/OpenSearchConnectionTest.java | 75 +++
.../stormcrawler/opensearch/WaitAckCacheTest.java | 336 +++++++++++++
.../opensearch/bolt/AbstractOpenSearchTest.java | 4 +-
.../opensearch/bolt/IndexerBoltTest.java | 16 +-
.../opensearch/bolt/QueueBoltTest.java | 202 ++++++++
.../opensearch/bolt/StatusBoltTest.java | 76 ++-
.../opensearch/metrics/MetricsReporterTest.java | 54 +++
.../opensearch/persistence/AbstractSpoutTest.java | 139 ++++++
.../src/test}/resources/indexer.mapping | 0
.../src/test}/resources/metrics.mapping | 0
.../src/test/resources/queues.mapping | 33 ++
.../src/test}/resources/status.mapping | 0
external/opensearch/README.md | 4 +-
external/opensearch/archetype/pom.xml | 2 +-
.../resources/archetype-resources/crawler.flux | 10 +
.../archetype-resources/docker-compose.yml | 12 +-
.../resources/archetype-resources/injection.flux | 11 +
.../archetype-resources/opensearch-conf.yaml | 9 +-
.../src/main/resources/archetype-resources/pom.xml | 6 +-
.../src/main/resources/queues.mapping | 33 ++
external/opensearch/opensearch-conf.yaml | 10 +-
external/opensearch/pom.xml | 10 +-
.../opensearch/BulkItemResponseToFailedFlag.java | 17 +-
.../apache/stormcrawler/opensearch/Constants.java | 1 +
.../stormcrawler/opensearch/DelegateRefresher.java | 172 +++++++
.../stormcrawler/opensearch/IndexCreation.java | 1 +
.../opensearch/OpenSearchConnection.java | 72 +--
.../stormcrawler/opensearch/WaitAckCache.java | 323 +++++++++++++
.../stormcrawler/opensearch/bolt/DeletionBolt.java | 222 ++-------
.../stormcrawler/opensearch/bolt/IndexerBolt.java | 258 ++--------
.../opensearch/filtering/JSONURLFilterWrapper.java | 124 +----
.../opensearch/metrics/MetricsConsumer.java | 9 +-
.../opensearch/metrics/MetricsReporter.java | 221 +++++++++
.../opensearch/metrics/StatusMetricsBolt.java | 12 +-
.../parse/filter/JSONResourceWrapper.java | 122 +----
.../opensearch/persistence/AbstractSpout.java | 22 +-
.../opensearch/persistence/AggregationSpout.java | 15 +-
.../opensearch/persistence/HybridSpout.java | 14 +-
.../opensearch/persistence/QueueBolt.java | 190 ++++++++
.../opensearch/persistence/StatusUpdaterBolt.java | 251 ++--------
.../opensearch/DelegateRefresherTest.java | 256 ++++++++++
.../stormcrawler/opensearch/WaitAckCacheTest.java | 329 +++++++++++++
.../opensearch/bolt/AbstractOpenSearchTest.java | 1 +
.../opensearch/bolt/IndexerBoltTest.java | 16 +-
.../{StatusBoltTest.java => QueueBoltTest.java} | 127 +++--
.../opensearch/bolt/StatusBoltTest.java | 35 +-
.../opensearch/metrics/MetricsReporterTest.java | 54 +++
.../opensearch/src/test/resources/queues.mapping | 33 ++
external/playwright/README.md | 198 ++++++++
external/playwright/playwright-conf.yaml | 19 +
external/playwright/pom.xml | 6 +-
.../protocol/playwright/HttpProtocol.java | 264 +++++++----
.../protocol/playwright/PageAction.java | 56 +++
.../protocol/playwright/PageActions.java | 156 ++++++
.../playwright/actions/DismissOverlayAction.java | 111 +++++
.../playwright/actions/EvaluateAction.java | 102 ++++
.../playwright/actions/ExpandClickablesAction.java | 152 ++++++
.../playwright/actions/ScreenshotAction.java | 107 +++++
.../playwright/actions/ScrollToBottomAction.java | 91 ++++
.../playwright/actions/WaitForSelectorAction.java | 122 +++++
.../bolt/JsRenderingRedirectionBolt.java | 130 +++++
.../parsefilter/JsRenderingDetector.java | 285 +++++++++++
.../protocol/playwright/LocalResourceHandler.java | 8 +-
.../protocol/playwright/PageActionsLiveTest.java | 323 +++++++++++++
.../protocol/playwright/PageActionsTest.java | 89 ++++
.../protocol/playwright/ProtocolTest.java | 19 +-
.../playwright/actions/ActionConfigureTest.java | 173 +++++++
.../bolt/JsRenderingRedirectionBoltTest.java | 153 ++++++
.../parsefilter/JsRenderingDetectorTest.java | 194 ++++++++
.../src/test/resources/page-actions-fixture.html | 64 +++
.../src/test/resources/page-actions.chain.json | 24 +
.../src/test/resources/page-actions.empty.json | 3 +
.../src/test/resources/page-actions.invalid.json | 9 +
.../src/test/resources/page-actions.live.json | 38 ++
.../src/test/resources/page-actions.single.json | 14 +
external/pom.xml | 2 +-
external/selenium/README.md | 2 +
external/selenium/pom.xml | 10 +-
.../protocol/selenium/NavigationFilter.java | 6 +
.../protocol/selenium/NavigationFilters.java | 50 +-
.../protocol/selenium/RemoteDriverProtocol.java | 13 +-
.../protocol/selenium/SeleniumProtocol.java | 6 +
.../protocol/selenium/ProtocolTest.java | 1 +
external/solr/README.md | 6 +-
external/solr/archetype/pom.xml | 2 +-
.../main/resources/archetype-resources/README.md | 2 +-
.../src/main/resources/archetype-resources/pom.xml | 6 +-
.../resources/archetype-resources/setup-solr.sh | 8 +-
external/solr/configsets/docs/conf/solrconfig.xml | 2 +-
.../solr/configsets/metrics/conf/solrconfig.xml | 2 +-
.../solr/configsets/status/conf/solrconfig.xml | 2 +-
external/solr/pom.xml | 87 +++-
.../org/apache/stormcrawler/solr/Constants.java | 1 +
.../apache/stormcrawler/solr/SolrConnection.java | 174 +++++--
.../stormcrawler/solr/bolt/DeletionBolt.java | 6 +-
.../apache/stormcrawler/solr/bolt/IndexerBolt.java | 11 +-
.../stormcrawler/solr/metrics/MetricsConsumer.java | 1 +
.../stormcrawler/solr/metrics/MetricsReporter.java | 174 +++++++
.../stormcrawler/solr/persistence/SolrSpout.java | 14 +-
.../solr/persistence/StatusUpdaterBolt.java | 1 +
.../stormcrawler/solr/CloudConnectionTest.java | 26 +-
.../stormcrawler/solr/SolrCloudContainerTest.java | 15 +-
.../stormcrawler/solr/SolrContainerTest.java | 19 +-
.../stormcrawler/solr/bolt/IndexerBoltTest.java | 17 +-
.../stormcrawler/solr/persistence/SpoutTest.java | 16 +-
.../solr/persistence/StatusBoltTest.java | 11 +-
.../solr/src/test/resources/docker-compose.yml | 8 +-
external/sql/pom.xml | 2 +-
.../org/apache/stormcrawler/sql/Constants.java | 1 +
.../org/apache/stormcrawler/sql/IndexerBolt.java | 8 +-
.../java/org/apache/stormcrawler/sql/SQLSpout.java | 37 +-
.../java/org/apache/stormcrawler/sql/SQLUtil.java | 1 +
.../apache/stormcrawler/sql/StatusUpdaterBolt.java | 24 +-
.../stormcrawler/sql/metrics/MetricsConsumer.java | 1 +
.../stormcrawler/sql/metrics/MetricsReporter.java | 180 +++++++
.../apache/stormcrawler/sql/AbstractSQLTest.java | 1 +
.../apache/stormcrawler/sql/IndexerBoltTest.java | 1 +
.../org/apache/stormcrawler/sql/SQLSpoutTest.java | 7 +-
.../stormcrawler/sql/StatusUpdaterBoltTest.java | 1 +
external/tika/pom.xml | 2 +-
.../org/apache/stormcrawler/tika/DOMBuilder.java | 10 +-
.../org/apache/stormcrawler/tika/ParserBolt.java | 12 +-
.../apache/stormcrawler/tika/RedirectionBolt.java | 1 +
.../stormcrawler/tika/XMLCharacterRecognizer.java | 14 +-
.../apache/stormcrawler/tika/ParserBoltTest.java | 13 +-
external/urlfrontier/README.md | 61 +++
external/urlfrontier/pom.xml | 17 +-
.../apache/stormcrawler/urlfrontier/Constants.java | 92 ++++
.../stormcrawler/urlfrontier/CrawlDelayPolicy.java | 216 +++++++++
.../stormcrawler/urlfrontier/HostBackoff.java | 310 ++++++++++++
.../urlfrontier/ManagedChannelUtil.java | 1 +
.../urlfrontier/QueueRegulatorBolt.java | 425 +++++++++++++++++
.../org/apache/stormcrawler/urlfrontier/Spout.java | 8 +-
.../urlfrontier/StatusUpdaterBolt.java | 20 +-
.../urlfrontier/CrawlDelayPolicyTest.java | 232 +++++++++
.../stormcrawler/urlfrontier/HostBackoffTest.java | 346 ++++++++++++++
.../QueueRegulatorBoltDecisionTest.java | 213 +++++++++
.../urlfrontier/QueueRegulatorBoltTest.java | 308 ++++++++++++
.../urlfrontier/StatusUpdaterBoltTest.java | 76 ++-
.../urlfrontier/URLFrontierContainer.java | 5 +-
.../urlfrontier/URLFrontierContainerConfig.java | 1 +
external/warc/README.md | 4 +-
external/warc/pom.xml | 27 +-
.../warc/FileTimeSizeRotationPolicy.java | 5 +-
.../org/apache/stormcrawler/warc/GzipHdfsBolt.java | 17 +-
.../stormcrawler/warc/MetadataRecordFormat.java | 23 +-
.../stormcrawler/warc/WARCFileNameFormat.java | 1 +
.../org/apache/stormcrawler/warc/WARCHdfsBolt.java | 8 +-
.../apache/stormcrawler/warc/WARCRecordFormat.java | 61 ++-
.../stormcrawler/warc/WARCRequestRecordFormat.java | 14 +-
.../org/apache/stormcrawler/warc/WARCSpout.java | 100 ++--
.../apache/stormcrawler/warc/WARCHdfsBoltTest.java | 43 +-
.../stormcrawler/warc/WARCRecordFormatTest.java | 105 ++--
.../apache/stormcrawler/warc/WARCSpoutTest.java | 1 +
pom.xml | 49 +-
368 files changed, 21093 insertions(+), 3174 deletions(-)
create mode 100644 AGENTS.md
create mode 100644 SECURITY.md
create mode 100644
core/src/main/java/org/apache/stormcrawler/metrics/CrawlerMetrics.java
copy core/src/{test/java/org/apache/stormcrawler/protocol/DummyProtocol.java
=> main/java/org/apache/stormcrawler/metrics/DualCounterMetric.java} (53%)
copy
external/ai/src/main/java/org/apache/stormcrawler/ai/listener/NoOpListener.java
=> core/src/main/java/org/apache/stormcrawler/metrics/DualReducedMetric.java
(52%)
copy core/src/main/java/org/apache/stormcrawler/{persistence/Status.java =>
metrics/ScopedCounter.java} (59%)
copy core/src/main/java/org/apache/stormcrawler/{proxy/ProxyManager.java =>
metrics/ScopedReducedMetric.java} (61%)
copy core/src/{test/java/org/apache/stormcrawler/protocol/DummyProtocol.java
=> main/java/org/apache/stormcrawler/metrics/v1/V1CounterMetric.java} (61%)
copy core/src/{test/java/org/apache/stormcrawler/protocol/DummyProtocol.java
=> main/java/org/apache/stormcrawler/metrics/v1/V1ReducedMetric.java} (60%)
create mode 100644
core/src/main/java/org/apache/stormcrawler/metrics/v2/V2CounterMetric.java
create mode 100644
core/src/main/java/org/apache/stormcrawler/metrics/v2/V2HistogramReducedMetric.java
create mode 100644
core/src/main/java/org/apache/stormcrawler/metrics/v2/V2MeterReducedMetric.java
create mode 100644
core/src/main/java/org/apache/stormcrawler/protocol/CIDR.java
create mode 100644
core/src/main/java/org/apache/stormcrawler/protocol/IPFilterRules.java
create mode 100644
core/src/main/java/org/apache/stormcrawler/proxy/ProxyMetadata.java
copy core/src/main/java/org/apache/stormcrawler/proxy/{ProxyManager.java =>
ProxyUtils.java} (57%)
create mode 100644
core/src/main/java/org/apache/stormcrawler/util/RetryAfterParser.java
create mode 100644
core/src/test/java/org/apache/stormcrawler/bolt/URLFilterBoltTest.java
create mode 100644
core/src/test/java/org/apache/stormcrawler/metrics/CrawlerMetricsTest.java
copy core/src/test/java/org/apache/stormcrawler/{MetadataTest.java =>
metrics/DualCounterMetricTest.java} (52%)
copy core/src/test/java/org/apache/stormcrawler/{MetadataTest.java =>
metrics/DualReducedMetricTest.java} (53%)
create mode 100644
core/src/test/java/org/apache/stormcrawler/metrics/v2/V2CounterMetricTest.java
create mode 100644
core/src/test/java/org/apache/stormcrawler/metrics/v2/V2HistogramReducedMetricTest.java
create mode 100644
core/src/test/java/org/apache/stormcrawler/metrics/v2/V2MeterReducedMetricTest.java
create mode 100644
core/src/test/java/org/apache/stormcrawler/protocol/CIDRTest.java
create mode 100644
core/src/test/java/org/apache/stormcrawler/protocol/IPFilterRulesTest.java
create mode 100644
core/src/test/java/org/apache/stormcrawler/protocol/RobotRulesParserAgentNamesTest.java
create mode 100644
core/src/test/java/org/apache/stormcrawler/protocol/httpclient/HttpClientProtocolProxyManagerTest.java
create mode 100644
core/src/test/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocolProxyConcurrencyTest.java
create mode 100644
core/src/test/java/org/apache/stormcrawler/util/RetryAfterParserTest.java
create mode 100644
core/src/test/java/org/apache/stormcrawler/util/URLUtilSanitizeTest.java
create mode 100644
core/src/test/java/org/apache/stormcrawler/util/URLUtilToURLTest.java
copy core/src/test/resources/{duplicateLinks.html => subdocuments.html} (60%)
create mode 100644 core/src/test/resources/test.urlfilterbolt.json
create mode 100644 core/src/test/resources/test.xpath.jsoupfilters.json
copy core/src/test/resources/{javascriptLinks.html => xpath-test.html} (57%)
create mode 100644 docs/src/main/asciidoc/extending.adoc
create mode 100644
external/aws/src/test/java/org/apache/stormcrawler/aws/s3/S3CacheBoltTest.java
create mode 100644 external/opensearch-java/README.md
copy external/{opensearch => opensearch-java}/pom.xml (81%)
create mode 100644
external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessor.java
create mode 100644
external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java
copy external/{opensearch =>
opensearch-java}/src/main/java/org/apache/stormcrawler/opensearch/Constants.java
(99%)
create mode 100644
external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java
copy external/{opensearch =>
opensearch-java}/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java
(54%)
create mode 100644
external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java
create mode 100644
external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/WaitAckCache.java
create mode 100644
external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java
create mode 100644
external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java
create mode 100644
external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java
copy external/{opensearch =>
opensearch-java}/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java
(84%)
create mode 100644
external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporter.java
create mode 100644
external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java
create mode 100644
external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java
copy external/{opensearch =>
opensearch-java}/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java
(62%)
create mode 100644
external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java
create mode 100644
external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java
create mode 100644
external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/QueueBolt.java
create mode 100644
external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java
create mode 100644
external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorTest.java
create mode 100644
external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/DelegateRefresherTest.java
create mode 100644
external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionTest.java
create mode 100644
external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java
copy external/{opensearch =>
opensearch-java}/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java
(94%)
copy external/{opensearch =>
opensearch-java}/src/test/java/org/apache/stormcrawler/opensearch/bolt/IndexerBoltTest.java
(90%)
create mode 100644
external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/QueueBoltTest.java
copy external/{opensearch =>
opensearch-java}/src/test/java/org/apache/stormcrawler/opensearch/bolt/StatusBoltTest.java
(66%)
create mode 100644
external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporterTest.java
create mode 100644
external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpoutTest.java
copy
external/{opensearch/archetype/src/main/resources/archetype-resources/src/main
=> opensearch-java/src/test}/resources/indexer.mapping (100%)
copy
external/{opensearch/archetype/src/main/resources/archetype-resources/src/main
=> opensearch-java/src/test}/resources/metrics.mapping (100%)
create mode 100644 external/opensearch-java/src/test/resources/queues.mapping
copy
external/{opensearch/archetype/src/main/resources/archetype-resources/src/main
=> opensearch-java/src/test}/resources/status.mapping (100%)
create mode 100644
external/opensearch/archetype/src/main/resources/archetype-resources/src/main/resources/queues.mapping
create mode 100644
external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java
create mode 100644
external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/WaitAckCache.java
create mode 100644
external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporter.java
create mode 100644
external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/QueueBolt.java
create mode 100644
external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/DelegateRefresherTest.java
create mode 100644
external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java
copy
external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/{StatusBoltTest.java
=> QueueBoltTest.java} (50%)
create mode 100644
external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporterTest.java
create mode 100644 external/opensearch/src/test/resources/queues.mapping
create mode 100644
external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/PageAction.java
create mode 100644
external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/PageActions.java
create mode 100644
external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/actions/DismissOverlayAction.java
create mode 100644
external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/actions/EvaluateAction.java
create mode 100644
external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/actions/ExpandClickablesAction.java
create mode 100644
external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/actions/ScreenshotAction.java
create mode 100644
external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/actions/ScrollToBottomAction.java
create mode 100644
external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/actions/WaitForSelectorAction.java
create mode 100644
external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/bolt/JsRenderingRedirectionBolt.java
create mode 100644
external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetector.java
create mode 100644
external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/PageActionsLiveTest.java
create mode 100644
external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/PageActionsTest.java
create mode 100644
external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/actions/ActionConfigureTest.java
create mode 100644
external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/bolt/JsRenderingRedirectionBoltTest.java
create mode 100644
external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetectorTest.java
create mode 100644
external/playwright/src/test/resources/page-actions-fixture.html
create mode 100644
external/playwright/src/test/resources/page-actions.chain.json
create mode 100644
external/playwright/src/test/resources/page-actions.empty.json
create mode 100644
external/playwright/src/test/resources/page-actions.invalid.json
create mode 100644
external/playwright/src/test/resources/page-actions.live.json
create mode 100644
external/playwright/src/test/resources/page-actions.single.json
create mode 100644
external/solr/src/main/java/org/apache/stormcrawler/solr/metrics/MetricsReporter.java
create mode 100644
external/sql/src/main/java/org/apache/stormcrawler/sql/metrics/MetricsReporter.java
create mode 100644
external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/CrawlDelayPolicy.java
create mode 100644
external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/HostBackoff.java
create mode 100644
external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/QueueRegulatorBolt.java
create mode 100644
external/urlfrontier/src/test/java/org/apache/stormcrawler/urlfrontier/CrawlDelayPolicyTest.java
create mode 100644
external/urlfrontier/src/test/java/org/apache/stormcrawler/urlfrontier/HostBackoffTest.java
create mode 100644
external/urlfrontier/src/test/java/org/apache/stormcrawler/urlfrontier/QueueRegulatorBoltDecisionTest.java
create mode 100644
external/urlfrontier/src/test/java/org/apache/stormcrawler/urlfrontier/QueueRegulatorBoltTest.java
