This is an automated email from the ASF dual-hosted git repository. lewismc pushed a commit to branch NUTCH-3162 in repository https://gitbox.apache.org/repos/asf/nutch.git
commit 1e83d8610f796823eec356321ade3fa5d7449fdc Merge: ba9c15a1b e9247b3e8 Author: lewismc <[email protected]> AuthorDate: Tue Jul 21 18:42:43 2026 -0700 Merge branch 'master' into NUTCH-3162 Resolve conflicts in IndexerMapReduce, TestIndexerMapReduce, and ReducerContextWrapper by keeping latency SequenceFile merge support alongside master's Mockito-based test wrapper and mapreduce.Counters. .asf.yaml | 28 +- .codespellrc | 5 + .github/dependabot.yml | 22 + .github/pull_request_template.md | 5 +- .github/workflows/junit-report.yml | 44 +- .github/workflows/master-build.yml | 179 ++- .github/workflows/sonarcloud.yml | 73 +- .github/workflows/yetus.yml | 116 ++ .gitignore | 2 + .markdownlint.yaml | 13 + src/test/domain-urlfilter.txt => .spectral.yaml | 19 +- .yamllint.yml | 10 + .yetus/blanks-eol.txt | 4 + .yetus/blanks-tabs.txt | 7 + .yetus/personality.sh | 40 + AGENTS.md | 188 +++ LICENSE-binary | 97 +- NOTICE-binary | 227 ++-- README.md | 132 +- SECURITY.md | 22 + THREAT_MODEL.md | 327 +++++ build.xml | 108 +- conf/log4j2.xml | 10 +- conf/nutch-default.xml | 56 +- default.properties | 19 +- docker/Dockerfile | 68 +- docker/README.md | 58 +- docker/config/supervisord_startserver.conf | 47 - docker/config/supervisord_startserver_webapp.conf | 69 - ivy/ivy.xml | 256 ++-- ivy/mvn.template | 4 +- openapi.yaml | 1326 ++++++++++++++++++++ src/bin/nutch | 5 - .../apache/nutch/crawl/AdaptiveFetchSchedule.java | 45 +- src/java/org/apache/nutch/crawl/CrawlDatum.java | 18 +- src/java/org/apache/nutch/crawl/CrawlDb.java | 4 +- src/java/org/apache/nutch/crawl/CrawlDbFilter.java | 35 +- src/java/org/apache/nutch/crawl/CrawlDbReader.java | 47 +- .../org/apache/nutch/crawl/CrawlDbReducer.java | 16 +- .../org/apache/nutch/crawl/DeduplicationJob.java | 23 +- src/java/org/apache/nutch/crawl/Generator.java | 76 +- src/java/org/apache/nutch/crawl/Injector.java | 11 +- src/java/org/apache/nutch/crawl/LinkDb.java | 27 +- src/java/org/apache/nutch/crawl/MD5Signature.java | 6 +- .../apache/nutch/crawl/TextProfileSignature.java | 7 +- src/java/org/apache/nutch/fetcher/FetchItem.java | 4 +- src/java/org/apache/nutch/fetcher/FetchNode.java | 72 -- src/java/org/apache/nutch/fetcher/FetchNodeDb.java | 48 - src/java/org/apache/nutch/fetcher/Fetcher.java | 27 +- .../apache/nutch/fetcher/FetcherOutputFormat.java | 3 +- .../org/apache/nutch/fetcher/FetcherThread.java | 109 +- src/java/org/apache/nutch/hostdb/HostDatum.java | 6 +- src/java/org/apache/nutch/hostdb/ReadHostDb.java | 9 +- .../org/apache/nutch/indexer/IndexerMapReduce.java | 79 +- .../apache/nutch/indexer/IndexerOutputFormat.java | 62 +- src/java/org/apache/nutch/indexer/IndexingJob.java | 4 +- src/java/org/apache/nutch/metadata/Nutch.java | 83 +- .../org/apache/nutch/metrics/NutchMetrics.java | 3 + .../org/apache/nutch/net/protocols/Response.java | 2 +- .../org/apache/nutch/parse/ParseOutputFormat.java | 85 +- src/java/org/apache/nutch/parse/ParseSegment.java | 24 +- src/java/org/apache/nutch/parse/ParseStatus.java | 6 +- .../apache/nutch/plugin/PluginManifestParser.java | 7 +- src/java/org/apache/nutch/protocol/Protocol.java | 13 + .../org/apache/nutch/protocol/ProtocolFactory.java | 48 +- .../apache/nutch/protocol/RobotRulesParser.java | 32 +- .../apache/nutch/scoring/webgraph/LinkRank.java | 53 +- .../nutch/segment/ContentAsTextInputFormat.java | 10 +- .../org/apache/nutch/segment/SegmentReader.java | 76 +- src/java/org/apache/nutch/service/ConfManager.java | 38 - src/java/org/apache/nutch/service/JobManager.java | 43 - src/java/org/apache/nutch/service/NutchReader.java | 38 - src/java/org/apache/nutch/service/NutchServer.java | 234 ---- src/java/org/apache/nutch/service/SeedManager.java | 32 - .../apache/nutch/service/impl/ConfManagerImpl.java | 138 -- .../org/apache/nutch/service/impl/JobFactory.java | 74 -- .../apache/nutch/service/impl/JobManagerImpl.java | 95 -- .../org/apache/nutch/service/impl/JobWorker.java | 117 -- .../org/apache/nutch/service/impl/LinkReader.java | 171 --- .../org/apache/nutch/service/impl/NodeReader.java | 179 --- .../service/impl/NutchServerPoolExecutor.java | 130 -- .../apache/nutch/service/impl/SeedManagerImpl.java | 61 - .../apache/nutch/service/impl/SequenceReader.java | 171 --- .../apache/nutch/service/impl/ServiceWorker.java | 50 - .../nutch/service/model/request/DbQuery.java | 56 - .../nutch/service/model/request/JobConfig.java | 73 -- .../nutch/service/model/request/NutchConfig.java | 51 - .../nutch/service/model/request/ReaderConfig.java | 30 - .../nutch/service/model/request/SeedList.java | 103 -- .../nutch/service/model/request/SeedUrl.java | 89 -- .../nutch/service/model/request/ServiceConfig.java | 51 - .../service/model/response/FetchNodeDbInfo.java | 107 -- .../nutch/service/model/response/JobInfo.java | 100 -- .../service/model/response/NutchServerInfo.java | 55 - .../nutch/service/model/response/ServiceInfo.java | 32 - .../nutch/service/resources/AbstractResource.java | 45 - .../nutch/service/resources/AdminResource.java | 87 -- .../nutch/service/resources/ConfigResource.java | 132 -- .../apache/nutch/service/resources/DbResource.java | 145 --- .../nutch/service/resources/JobResource.java | 99 -- .../nutch/service/resources/ReaderResouce.java | 177 --- .../nutch/service/resources/SeedResource.java | 113 -- .../nutch/service/resources/ServicesResource.java | 83 -- .../nutch/tools/AbstractCommonCrawlFormat.java | 4 +- .../apache/nutch/tools/CommonCrawlDataDumper.java | 17 +- .../nutch/tools/CommonCrawlFormatJackson.java | 33 +- .../apache/nutch/tools/CommonCrawlFormatWARC.java | 9 +- src/java/org/apache/nutch/tools/DmozParser.java | 40 +- src/java/org/apache/nutch/tools/FileDumper.java | 50 +- src/java/org/apache/nutch/tools/ResolveUrls.java | 13 +- .../org/apache/nutch/tools/ShowProperties.java | 22 +- .../org/apache/nutch/tools/arc/ArcInputFormat.java | 2 +- .../apache/nutch/tools/arc/ArcRecordReader.java | 43 +- .../apache/nutch/tools/arc/ArcSegmentCreator.java | 44 +- .../org/apache/nutch/tools/warc/WARCExporter.java | 27 +- .../org/apache/nutch/util/AbstractChecker.java | 16 +- src/java/org/apache/nutch/util/DumpFileUtil.java | 47 +- .../org/apache/nutch/util/EncodingDetector.java | 25 +- src/java/org/apache/nutch/util/JexlUtil.java | 151 ++- src/java/org/apache/nutch/util/NutchJob.java | 18 +- .../org/apache/nutch/util/PrefixStringMatcher.java | 4 +- .../org/apache/nutch/util/SitemapProcessor.java | 12 +- src/java/org/apache/nutch/util/TimingUtil.java | 17 +- src/java/org/apache/nutch/util/URLUtil.java | 230 +++- src/plugin/build-plugin.xml | 28 + src/plugin/build.xml | 13 + .../apache/nutch/exchange/jexl/JexlExchange.java | 3 +- .../org/apache/nutch/parse/feed/FeedParser.java | 44 +- src/plugin/headings/ivy.xml | 4 - src/plugin/headings/plugin.xml | 1 + .../parse/headings/TestHeadingsParseFilter.java | 19 +- .../nutch/indexer/anchor/AnchorIndexingFilter.java | 7 +- .../nutch/indexer/jexl/JexlIndexingFilter.java | 2 +- .../nutch/indexer/links/LinksIndexingFilter.java | 22 +- .../cloudsearch/CloudSearchIndexWriter.java | 7 +- .../nutch/indexwriter/csv/CSVIndexWriter.java | 12 +- .../nutch/indexwriter/dummy/DummyIndexWriter.java | 7 +- .../nutch/indexwriter/rabbit/RabbitDocument.java | 7 +- .../indexwriter/rabbit/RabbitIndexWriter.java | 20 +- .../nutch/indexwriter/rabbit/RabbitMessage.java | 7 +- src/plugin/language-identifier/ivy.xml | 2 +- src/plugin/language-identifier/plugin.xml | 18 +- .../nutch/analysis/lang/HTMLLanguageParser.java | 7 +- .../analysis/lang/TestHTMLLanguageParser.java | 11 +- .../apache/nutch/protocol/http/api/HttpBase.java | 48 +- .../protocol/http/api/HttpRobotRulesParser.java | 24 +- .../protocol/http/api/TestRobotRulesParser.java | 38 +- src/plugin/lib-nekohtml/ivy.xml | 2 +- src/plugin/lib-nekohtml/plugin.xml | 2 +- .../nutch/urlfilter/api/RegexURLFilterBase.java | 58 +- .../urlfilter/api/RegexURLFilterBaseTest.java | 28 +- src/plugin/lib-xml/ivy.xml | 6 +- .../nutch/microformats/reltag/RelTagParser.java | 23 +- .../indexer/filter/MimeTypeIndexingFilter.java | 47 +- .../indexer/filter/MimeTypeIndexingFilterTest.java | 13 +- .../java/org/apache/nutch/parse/ext/ExtParser.java | 46 +- .../apache/nutch/parse/html/DOMContentUtils.java | 55 +- .../apache/nutch/parse/html/HTMLMetaProcessor.java | 16 +- .../nutch/parse/html/TestDOMContentUtils.java | 22 +- .../nutch/parse/html/TestRobotsMetaProcessor.java | 34 +- .../org/apache/nutch/parse/js/JSParseFilter.java | 27 +- src/plugin/parse-tika/ivy.xml | 4 +- src/plugin/parse-tika/plugin.xml | 132 +- .../org/apache/nutch/parse/tika/DOMBuilder.java | 150 ++- .../apache/nutch/parse/tika/DOMContentUtils.java | 48 +- .../nutch/parse/tika/TestBoilerpipeExtraction.java | 49 +- .../parse/tika/TestLinkExtractionEdgeCases.java | 69 +- .../nutch/parse/tika/TestMetadataExtraction.java | 59 +- .../org/apache/nutch/parse/tika/TestOOParser.java | 6 +- .../parse/tika/TestParserFailureHandling.java | 68 +- .../nutch/parse/tika/TestRobotsMetaProcessor.java | 21 +- .../nutch/parsefilter/naivebayes/Classify.java | 11 +- .../apache/nutch/parsefilter/naivebayes/Train.java | 7 +- .../nutch/parsefilter/regex/RegexParseFilter.java | 43 +- .../parsefilter/regex/TestRegexParseFilter.java | 24 +- .../java/org/apache/nutch/protocol/file/File.java | 15 +- .../apache/nutch/protocol/file/FileResponse.java | 37 +- .../java/org/apache/nutch/protocol/foo/Foo.java | 11 +- src/plugin/protocol-ftp/ivy.xml | 3 +- .../java/org/apache/nutch/protocol/ftp/Client.java | 63 +- .../java/org/apache/nutch/protocol/ftp/Ftp.java | 42 +- .../org/apache/nutch/protocol/ftp/FtpResponse.java | 46 +- .../nutch/protocol/ftp/FtpRobotRulesParser.java | 10 +- .../apache/nutch/protocol/ftp/FtpProtocolIT.java | 159 +++ .../nutch/protocol/htmlunit/HttpResponse.java | 9 +- .../protocol/htmlunit/HtmlUnitProtocolIT.java | 79 ++ .../apache/nutch/protocol/http/HttpResponse.java | 12 +- .../apache/nutch/protocol/http/HttpProtocolIT.java | 83 ++ .../protocol/http/TestBadServerResponses.java | 15 +- src/plugin/protocol-httpclient/ivy.xml | 1 + .../httpclient/HttpBasicAuthentication.java | 28 +- .../nutch/protocol/httpclient/HttpResponse.java | 11 + .../protocol/httpclient/HttpClientProtocolIT.java | 150 +++ .../httpclient/TestProtocolHttpClient.java | 35 +- .../protocol/interactiveselenium/HttpResponse.java | 18 +- src/plugin/protocol-okhttp/ivy.xml | 6 +- src/plugin/protocol-okhttp/plugin.xml | 14 +- .../org/apache/nutch/protocol/okhttp/CIDR.java | 38 +- .../nutch/protocol/okhttp/IPFilterRules.java | 11 +- .../org/apache/nutch/protocol/okhttp/OkHttp.java | 23 + .../nutch/protocol/okhttp/OkHttpResponse.java | 29 +- .../nutch/protocol/okhttp/OkHttpProtocolIT.java | 79 ++ .../protocol/okhttp/TestBadServerResponses.java | 33 +- .../protocol/okhttp/TestIPAddressFiltering.java | 32 +- .../nutch/protocol/selenium/HttpResponse.java | 13 +- .../protocol/selenium/SeleniumProtocolIT.java | 86 ++ .../publisher/rabbitmq/RabbitMQPublisherImpl.java | 3 +- .../subcollection/SubcollectionIndexingFilter.java | 27 +- .../apache/nutch/collection/TestSubcollection.java | 12 +- .../nutch/urlfilter/domain/DomainURLFilter.java | 23 +- .../domaindenylist/DomainDenylistURLFilter.java | 21 +- .../apache/nutch/urlfilter/fast/FastURLFilter.java | 56 +- .../nutch/urlfilter/prefix/PrefixURLFilter.java | 33 +- .../nutch/urlfilter/suffix/SuffixURLFilter.java | 61 +- .../net/urlnormalizer/ajax/AjaxURLNormalizer.java | 26 +- .../urlnormalizer/basic/BasicURLNormalizer.java | 59 +- .../basic/TestBasicURLNormalizer.java | 66 + .../net/urlnormalizer/host/HostURLNormalizer.java | 15 +- .../protocol/ProtocolURLNormalizer.java | 18 +- .../urlnormalizer/regex/RegexURLNormalizer.java | 24 +- .../regex/TestRegexURLNormalizer.java | 29 +- .../urlnormalizer/slash/SlashURLNormalizer.java | 32 +- src/test/host-protocol-mapping.txt | 35 + .../org/apache/nutch/crawl/CrawlDBTestUtil.java | 115 +- .../org/apache/nutch/crawl/CrawlDbUpdateUtil.java | 8 +- .../crawl/TestCrawlDbFilterExceptionHandling.java | 167 +++ .../org/apache/nutch/crawl/TestCrawlDbStates.java | 387 ++++-- .../nutch/crawl/TestCrawlDbStatesExtended.java | 195 --- .../nutch/fetcher/TestFetchWithParseFailures.java | 280 +++++ src/test/org/apache/nutch/fetcher/TestFetcher.java | 36 +- .../apache/nutch/indexer/TestIndexerMapReduce.java | 70 +- .../org/apache/nutch/metrics/LatencyTestUtil.java | 2 +- .../org/apache/nutch/plugin/TestPluginSystem.java | 26 +- .../nutch/protocol/AbstractProtocolPluginIT.java | 97 ++ .../protocol/ProtocolPluginIntegrationTest.java | 50 + .../apache/nutch/protocol/TestProtocolFactory.java | 30 +- .../apache/nutch/segment/TestSegmentMerger.java | 9 +- .../segment/TestSegmentMergerCrawlDatums.java | 9 +- .../org/apache/nutch/service/TestNutchServer.java | 66 - .../apache/nutch/util/ReducerContextWrapper.java | 417 +----- .../apache/nutch/util/TestEncodingDetector.java | 22 +- src/test/org/apache/nutch/util/TestGZIPUtils.java | 29 +- src/test/org/apache/nutch/util/TestJexlUtil.java | 160 +++ src/test/org/apache/nutch/util/TestMimeUtil.java | 5 +- src/test/org/apache/nutch/util/TestNodeWalker.java | 5 +- src/test/org/apache/nutch/util/TestURLUtil.java | 142 ++- .../fetch-parse-failure/index.html} | 31 +- .../fetch-parse-failure/robots.txt} | 12 +- .../fetch-parse-failure/test.html} | 31 +- src/testresources/forbidden-apis-signatures.txt | 24 + 250 files changed, 7592 insertions(+), 6774 deletions(-) diff --cc src/java/org/apache/nutch/indexer/IndexerMapReduce.java index 5ffde8aa2,448316e08..b4fc737f4 --- a/src/java/org/apache/nutch/indexer/IndexerMapReduce.java +++ b/src/java/org/apache/nutch/indexer/IndexerMapReduce.java @@@ -21,28 -22,19 +22,26 @@@ import java.nio.charset.Charset import java.util.Collection; import java.util.Locale; - import org.apache.hadoop.io.BytesWritable; - import org.apache.hadoop.io.NullWritable; - import org.apache.hadoop.io.SequenceFile; - import org.apache.hadoop.fs.FileSystem; - import org.apache.hadoop.fs.Path; - import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; - import org.slf4j.Logger; - import org.slf4j.LoggerFactory; import org.apache.commons.codec.binary.Base64; import org.apache.commons.codec.binary.StringUtils; - import org.apache.hadoop.conf.Configured; import org.apache.hadoop.conf.Configuration; + import org.apache.hadoop.conf.Configured; ++import org.apache.hadoop.fs.FileSystem; + import org.apache.hadoop.fs.Path; ++import org.apache.hadoop.io.BytesWritable; ++import org.apache.hadoop.io.IntWritable; ++import org.apache.hadoop.io.NullWritable; ++import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapreduce.Counter; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; + import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; ++import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat; import org.apache.nutch.crawl.CrawlDatum; import org.apache.nutch.crawl.CrawlDb; import org.apache.nutch.crawl.Inlinks; @@@ -62,9 -55,10 +62,12 @@@ import org.apache.nutch.parse.ParseText import org.apache.nutch.protocol.Content; import org.apache.nutch.scoring.ScoringFilterException; import org.apache.nutch.scoring.ScoringFilters; + import org.apache.nutch.segment.SegmentReader; + import org.slf4j.Logger; + import org.slf4j.LoggerFactory; +import com.tdunning.math.stats.MergingDigest; + /** * <p> * This class is typically invoked from within diff --cc src/test/org/apache/nutch/indexer/TestIndexerMapReduce.java index c2671bd66,3c2b68362..dc3bcafda --- a/src/test/org/apache/nutch/indexer/TestIndexerMapReduce.java +++ b/src/test/org/apache/nutch/indexer/TestIndexerMapReduce.java @@@ -55,7 -47,7 +56,8 @@@ import java.util.Map import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; + import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; /** Test {@link IndexerMapReduce} */ public class TestIndexerMapReduce { diff --cc src/test/org/apache/nutch/metrics/LatencyTestUtil.java index 0b4bc4f3c,000000000..093c8396e mode 100644,000000..100644 --- a/src/test/org/apache/nutch/metrics/LatencyTestUtil.java +++ b/src/test/org/apache/nutch/metrics/LatencyTestUtil.java @@@ -1,142 -1,0 +1,142 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nutch.metrics; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.io.BytesWritable; - import org.apache.hadoop.mapred.Counters; ++import org.apache.hadoop.mapreduce.Counters; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Test utility for latency-tracking tests. Reduces boilerplate when testing + * Fetcher, ParseSegment, and Indexer reducers that merge TDigests and set + * job-level percentile counters. + * + * <p>Use with the real Hadoop {@link Counters} from {@link org.apache.nutch.util.ReducerContextWrapper#getCounters()} + * (no mocks). + */ +public final class LatencyTestUtil { + + private static final String DUMMY_GROUP = "test"; + private static final String DUMMY_PREFIX = "latency"; + + private LatencyTestUtil() {} + + /** + * Builds serialized TDigest bytes from the given samples. Uses a temporary + * LatencyTracker with dummy group/prefix. Callers wrap the result as needed + * (e.g. {@code new BytesWritable(bytes)} or {@code new NutchWritable(new BytesWritable(bytes))}). + * + * @param samples latency values in milliseconds to record + * @return serialized digest as from {@link LatencyTracker#toBytes()} + */ + public static byte[] createDigestBytes(long... samples) { + LatencyTracker tracker = new LatencyTracker(DUMMY_GROUP, DUMMY_PREFIX); + for (long sample : samples) { + tracker.record(sample); + } + return tracker.toBytes(); + } + + /** + * Builds one BytesWritable per array of samples (e.g. one per map task). + * Useful for reducer tests that merge multiple digests. + * + * @param sampleArrays each array is recorded into one tracker and serialized to one BytesWritable + * @return list of digest BytesWritable, in order + */ + public static List<BytesWritable> createDigestBytesWritables(long[]... sampleArrays) { + List<BytesWritable> list = new ArrayList<>(sampleArrays.length); + for (long[] samples : sampleArrays) { + list.add(new BytesWritable(createDigestBytes(samples))); + } + return list; + } + + /** + * Asserts that the job-level percentile counters (p50, p95, p99) for the + * given group and prefix are in the range [minMs, maxMs]. Uses + * {@link LatencyTracker#SUFFIX_P50_MS} etc. + * + * @param counters counters from {@link org.apache.nutch.util.ReducerContextWrapper#getCounters()} + * @param group counter group (e.g. {@link NutchMetrics#GROUP_FETCHER}) + * @param prefix counter name prefix (e.g. {@link NutchMetrics#FETCHER_LATENCY}) + * @param minMs inclusive lower bound for all percentiles (ms) + * @param maxMs inclusive upper bound for all percentiles (ms) + */ + public static void assertPercentilesInRange(Counters counters, String group, String prefix, + long minMs, long maxMs) { + long p50 = counters.findCounter(group, prefix + LatencyTracker.SUFFIX_P50_MS).getValue(); + long p95 = counters.findCounter(group, prefix + LatencyTracker.SUFFIX_P95_MS).getValue(); + long p99 = counters.findCounter(group, prefix + LatencyTracker.SUFFIX_P99_MS).getValue(); + assertTrue(p50 >= minMs && p50 <= maxMs, + "p50=" + p50 + " not in [" + minMs + "," + maxMs + "]"); + assertTrue(p95 >= minMs && p95 <= maxMs, + "p95=" + p95 + " not in [" + minMs + "," + maxMs + "]"); + assertTrue(p99 >= minMs && p99 <= maxMs, + "p99=" + p99 + " not in [" + minMs + "," + maxMs + "]"); + } + + /** + * Asserts that the count and sum counters for the given group and prefix + * match the expected values. Uses {@link LatencyTracker#SUFFIX_COUNT_TOTAL} + * and {@link LatencyTracker#SUFFIX_SUM_MS}. + * + * @param counters counters from {@link org.apache.nutch.util.ReducerContextWrapper#getCounters()} + * @param group counter group + * @param prefix counter name prefix + * @param expectedCount expected _count_total value + * @param expectedSumMs expected _sum_ms value + */ + public static void assertCountAndSum(Counters counters, String group, String prefix, + long expectedCount, long expectedSumMs) { + assertEquals(expectedCount, + counters.findCounter(group, prefix + LatencyTracker.SUFFIX_COUNT_TOTAL).getValue()); + assertEquals(expectedSumMs, + counters.findCounter(group, prefix + LatencyTracker.SUFFIX_SUM_MS).getValue()); + } + + /** + * Asserts that the percentile counters (p50, p95, p99) for the given group + * and prefix are all zero. Useful for tests that emit with zero samples. + * + * @param counters counters from {@link org.apache.nutch.util.ReducerContextWrapper#getCounters()} + * @param group counter group + * @param prefix counter name prefix + */ + public static void assertPercentilesZero(Counters counters, String group, String prefix) { + assertEquals(0, counters.findCounter(group, prefix + LatencyTracker.SUFFIX_P50_MS).getValue()); + assertEquals(0, counters.findCounter(group, prefix + LatencyTracker.SUFFIX_P95_MS).getValue()); + assertEquals(0, counters.findCounter(group, prefix + LatencyTracker.SUFFIX_P99_MS).getValue()); + } + + /** + * Asserts that count, sum, and all percentile counters are zero. + * + * @param counters counters from {@link org.apache.nutch.util.ReducerContextWrapper#getCounters()} + * @param group counter group + * @param prefix counter name prefix + */ + public static void assertCountSumAndPercentilesZero(Counters counters, String group, String prefix) { + assertCountAndSum(counters, group, prefix, 0, 0); + assertPercentilesZero(counters, group, prefix); + } +} diff --cc src/test/org/apache/nutch/util/ReducerContextWrapper.java index c50853109,ec683e466..8fddf5857 --- a/src/test/org/apache/nutch/util/ReducerContextWrapper.java +++ b/src/test/org/apache/nutch/util/ReducerContextWrapper.java @@@ -91,327 -71,34 +71,44 @@@ public class ReducerContextWrapper<KEYI return context; } + /** + * Return the underlying counters updated by the context, for assertions in tests. - * Uses the real Hadoop mapred Counters API (no mocks). ++ * Uses the real Hadoop Counters API (no mocks). + * + * @return the counters instance + */ + public Counters getCounters() { + return counters; + } + + @SuppressWarnings("unchecked") private void initContext() { - // most methods are not used in Nutch unit tests. - context = reducer.new Context() { - - @Override - public KEYIN getCurrentKey() throws IOException, InterruptedException { - return currentKey; - } - - @Override - public VALUEIN getCurrentValue() throws IOException, InterruptedException { - return currentValue; - } - - @Override - public boolean nextKeyValue() throws IOException, InterruptedException { - return valuesIndex < valuesIn.size(); - } - - @SuppressWarnings("unchecked") - @Override - public void write(Object arg0, Object arg1) - throws IOException, InterruptedException { - valuesOut.put((KEYOUT) arg0, (VALUEOUT) arg1); - currentKey = (KEYIN) arg0; - currentValue = (VALUEIN) arg1; - valuesIndex++; - } - - @Override - public Counter getCounter(Enum<?> arg0) { - return counters.findCounter(arg0); - } - - @Override - public Counter getCounter(String arg0, String arg1) { - return counters.findCounter(arg0, arg1); - } - - @Override - public float getProgress() { - return valuesIndex; - } - - @Override - public String getStatus() { - return status; - } - - @Override - public void setStatus(String arg0) { - status = arg0; - } - - @Override - public Configuration getConfiguration() { - return config; - } - - @Override - public Iterable<VALUEIN> getValues() - throws IOException, InterruptedException { - return valuesIn.values(); - } - - @Override - public boolean nextKey() throws IOException, InterruptedException { - return valuesIndex < valuesIn.size(); - } - - @Override - public OutputCommitter getOutputCommitter() { - // Auto-generated - return null; - } - - @Override - public TaskAttemptID getTaskAttemptID() { - // Auto-generated - return null; - } - - @Override - public Path[] getArchiveClassPaths() { - // Auto-generated - return null; - } - - @Override - public String[] getArchiveTimestamps() { - // Auto-generated - return null; - } - - @Override - public URI[] getCacheArchives() throws IOException { - // Auto-generated - return null; - } - - @Override - public URI[] getCacheFiles() throws IOException { - // Auto-generated - return null; - } - - @Override - public Class<? extends Reducer<?, ?, ?, ?>> getCombinerClass() - throws ClassNotFoundException { - // Auto-generated - return null; - } - - @Override - public RawComparator<?> getCombinerKeyGroupingComparator() { - // Auto-generated - return null; - } - - @Override - public Credentials getCredentials() { - // Auto-generated - return null; - } - - @Override - public Path[] getFileClassPaths() { - // Auto-generated - return null; - } - - @Override - public String[] getFileTimestamps() { - // Auto-generated - return null; - } - - @Override - public RawComparator<?> getGroupingComparator() { - // Auto-generated - return null; - } - - @Override - public Class<? extends InputFormat<?, ?>> getInputFormatClass() - throws ClassNotFoundException { - // Auto-generated - return null; - } - - @Override - public String getJar() { - // Auto-generated - return null; - } - - @Override - public JobID getJobID() { - // Auto-generated - return null; - } - - @Override - public String getJobName() { - // Auto-generated - return null; - } - - @Override - public boolean getJobSetupCleanupNeeded() { - // Auto-generated - return false; - } - - @Override - public Path[] getLocalCacheArchives() throws IOException { - // Auto-generated - return null; - } - - @Override - public Path[] getLocalCacheFiles() throws IOException { - // Auto-generated - return null; - } - - @Override - public Class<?> getMapOutputKeyClass() { - // Auto-generated - return null; - } - - @Override - public Class<?> getMapOutputValueClass() { - // Auto-generated - return null; - } - - @Override - public Class<? extends Mapper<?, ?, ?, ?>> getMapperClass() - throws ClassNotFoundException { - // Auto-generated - return null; - } - - @Override - public int getMaxMapAttempts() { - // Auto-generated - return 0; - } - - @Override - public int getMaxReduceAttempts() { - // Auto-generated - return 0; - } - - @Override - public int getNumReduceTasks() { - // Auto-generated - return 0; - } - - @Override - public Class<? extends OutputFormat<?, ?>> getOutputFormatClass() - throws ClassNotFoundException { - // Auto-generated - return null; - } - - @Override - public Class<?> getOutputKeyClass() { - // Auto-generated - return null; - } - - @Override - public Class<?> getOutputValueClass() { - // Auto-generated - return null; - } - - @Override - public Class<? extends Partitioner<?, ?>> getPartitionerClass() - throws ClassNotFoundException { - // Auto-generated - return null; - } - - @Override - public boolean getProfileEnabled() { - // Auto-generated - return false; - } - - @Override - public String getProfileParams() { - // Auto-generated - return null; - } - - @Override - public IntegerRanges getProfileTaskRange(boolean arg0) { - // Auto-generated - return null; - } - - @Override - public Class<? extends Reducer<?, ?, ?, ?>> getReducerClass() - throws ClassNotFoundException { - // Auto-generated - return null; - } - - @Override - public RawComparator<?> getSortComparator() { - // Auto-generated - return null; - } - - @Override - public boolean getSymlink() { - // Auto-generated - return false; - } - - @Override - public boolean getTaskCleanupNeeded() { - // Auto-generated - return false; - } - - @Override - public String getUser() { - // Auto-generated - return null; - } - - @Override - public Path getWorkingDirectory() throws IOException { - // Auto-generated - return null; - } - - @Override - public void progress() { - // Auto-generated - } - }; - + context = Mockito.mock(Reducer.Context.class, + Mockito.withSettings().defaultAnswer(Mockito.RETURNS_DEFAULTS)); + + Mockito.lenient().when(context.getConfiguration()).thenReturn(config); + + Mockito.lenient().when(context.getCounter(ArgumentMatchers.<Enum<?>>any())) + .thenAnswer(inv -> counters.findCounter(inv.getArgument(0, Enum.class))); + + Mockito.lenient().when(context.getCounter(Mockito.anyString(), Mockito.anyString())) + .thenAnswer(inv -> counters.findCounter( + inv.getArgument(0, String.class), inv.getArgument(1, String.class))); + + try { + Mockito.doAnswer(new Answer<Void>() { + @Override + @SuppressWarnings("unchecked") + public Void answer(InvocationOnMock inv) { + KEYOUT k = inv.getArgument(0); + VALUEOUT v = inv.getArgument(1); + valuesOut.put(k, v); + return null; + } + }).when(context).write(Mockito.any(), Mockito.any()); + } catch (IOException | InterruptedException e) { + throw new IllegalStateException(e); + } } - }
