This is an automated email from the ASF dual-hosted git repository.
lewismc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nutch.git
The following commit(s) were added to refs/heads/master by this push:
new afbebd37c NUTCH-3207 Unify indentation in nutch-default.xml (#955)
afbebd37c is described below
commit afbebd37cad5539984d7201c2207b6e2a3697d8d
Author: Sebastian Nagel <[email protected]>
AuthorDate: Thu Aug 27 21:18:53 2026 +0200
NUTCH-3207 Unify indentation in nutch-default.xml (#955)
---
conf/nutch-default.xml | 1935 ++++++++++++++++++++++++++++--------------------
1 file changed, 1123 insertions(+), 812 deletions(-)
diff --git a/conf/nutch-default.xml b/conf/nutch-default.xml
index 949716dac..45d45163a 100644
--- a/conf/nutch-default.xml
+++ b/conf/nutch-default.xml
@@ -27,20 +27,24 @@
<property>
<name>file.content.limit</name>
<value>1048576</value>
- <description>The length limit for downloaded content using the file://
- protocol, in bytes. If this value is non-negative (>=0), content longer
- than it will be truncated; otherwise, no truncation at all. Do not
- confuse this setting with the http.content.limit setting.
+ <description>
+ The length limit for downloaded content using the file://
+ protocol, in bytes. If this value is non-negative (>=0), content longer
+ than it will be truncated; otherwise, no truncation at all. Do not
+ confuse this setting with the http.content.limit setting.
</description>
</property>
<property>
<name>file.crawl.parent</name>
<value>true</value>
- <description>The crawler is not restricted to the directories that you
specified in the
- URLs file but it is jumping into the parent directories as well. For your
own crawlings you can
- change this behavior (set to false) the way that only directories beneath
the directories that you specify get
- crawled.</description>
+ <description>
+ The crawler is not restricted to the directories that you
+ specified in the URLs file but it is jumping into the parent
+ directories as well. For your own crawlings you can change this
+ behavior (set to false) the way that only directories beneath the
+ directories that you specify get crawled.
+ </description>
</property>
<property>
@@ -59,11 +63,12 @@
<property>
<name>file.content.ignored</name>
<value>true</value>
- <description>If true, no file content will be saved during fetch.
- And it is probably what we want to set most of time, since file:// URLs
- are meant to be local and we can always use them directly at parsing
- and indexing stages. Otherwise file contents will be saved.
- !! NOT IMPLEMENTED YET !!
+ <description>
+ If true, no file content will be saved during fetch.
+ And it is probably what we want to set most of time, since file:// URLs
+ are meant to be local and we can always use them directly at parsing
+ and indexing stages. Otherwise file contents will be saved.
+ !! NOT IMPLEMENTED YET !!
</description>
</property>
@@ -72,130 +77,144 @@
<property>
<name>http.agent.name</name>
<value></value>
- <description>'User-Agent' name: a single word uniquely identifying your
crawler.
+ <description>
+ 'User-Agent' name: a single word uniquely identifying your crawler.
- The value is used to select the group of robots.txt rules addressing your
- crawler. It is also sent as part of the HTTP 'User-Agent' request header.
+ The value is used to select the group of robots.txt rules addressing your
+ crawler. It is also sent as part of the HTTP 'User-Agent' request header.
- This property MUST NOT be empty -
- please set this to a single word uniquely related to your organization.
+ This property MUST NOT be empty -
+ please set this to a single word uniquely related to your organization.
- Following RFC 9309 the 'User-Agent' name (aka. 'product token')
- "MUST contain only uppercase and lowercase letters ('a-z' and
- 'A-Z'), underscores ('_'), and hyphens ('-')."
+ Following RFC 9309 the 'User-Agent' name (aka. 'product token')
+ "MUST contain only uppercase and lowercase letters ('a-z' and
+ 'A-Z'), underscores ('_'), and hyphens ('-')."
- NOTE: You should also check other related properties:
+ NOTE: You should also check other related properties:
- http.robots.agents
- http.agent.description
- http.agent.url
- http.agent.email
- http.agent.version
+ http.robots.agents
+ http.agent.description
+ http.agent.url
+ http.agent.email
+ http.agent.version
- and set their values appropriately.
+ and set their values appropriately.
</description>
</property>
<property>
<name>http.robots.agents</name>
<value></value>
- <description>Any other agents, apart from 'http.agent.name', that the robots
- parser would look for in robots.txt. Multiple agents can be provided using
- comma as a delimiter. eg. mybot,foo-spider,bar-crawler
+ <description>
+ Any other agents, apart from 'http.agent.name', that the robots
+ parser would look for in robots.txt. Multiple agents can be provided using
+ comma as a delimiter. eg. mybot,foo-spider,bar-crawler
- The ordering of agents does NOT matter and the robots.txt parser combines
- all rules to any of the agent names. Also, there is NO need to add
- a wildcard (ie. "*") to this string as the robots parser would smartly
- take care of a no-match situation.
+ The ordering of agents does NOT matter and the robots.txt parser combines
+ all rules to any of the agent names. Also, there is NO need to add
+ a wildcard (ie. "*") to this string as the robots parser would smartly
+ take care of a no-match situation.
- If no value is specified, by default HTTP agent (ie. 'http.agent.name')
- is used for user-agent matching by the robots parser.
+ If no value is specified, by default HTTP agent (ie. 'http.agent.name')
+ is used for user-agent matching by the robots parser.
</description>
</property>
<property>
<name>http.robot.rules.allowlist</name>
<value></value>
- <description>Comma separated list of hostnames or IP addresses to ignore
- robot rules parsing for. Use with care and only if you are explicitly
- allowed by the site owner to ignore the site's robots.txt!
- Also keep in mind: ignoring the robots.txt rules means that no robots.txt
- file is requested which may cause the following side effects:
- (1) undesired content (duplicates, private pages, etc.) excluded via
+ <description>
+ Comma separated list of hostnames or IP addresses to ignore
+ robot rules parsing for. Use with care and only if you are explicitly
+ allowed by the site owner to ignore the site's robots.txt!
+ Also keep in mind: ignoring the robots.txt rules means that no robots.txt
+ file is requested which may cause the following side effects:
+ (1) undesired content (duplicates, private pages, etc.) excluded via
robotst.txt rules is crawled,
- (2) the Crawl-Delay directive is ignored and
- (3) no sitemaps are detected.
+ (2) the Crawl-Delay directive is ignored and
+ (3) no sitemaps are detected.
</description>
</property>
<property>
<name>http.robots.403.allow</name>
<value>true</value>
- <description>Some servers return HTTP status 403 (Forbidden) if
- /robots.txt doesn't exist. This should probably mean that we are
- allowed to crawl the site nonetheless. If this is set to false,
- then such sites will be treated as forbidden.</description>
+ <description>
+ Some servers return HTTP status 403 (Forbidden) if
+ /robots.txt doesn't exist. This should probably mean that we are
+ allowed to crawl the site nonetheless. If this is set to false,
+ then such sites will be treated as forbidden.
+ </description>
</property>
<property>
<name>http.robots.503.defer.visits</name>
<value>true</value>
- <description>Temporarily suspend fetching from a host if the
- robots.txt response is HTTP 503 or any other 5xx server error
- and HTTP 429 Too Many Requests. See also
- http.robots.503.defer.visits.delay and
- http.robots.503.defer.visits.retries</description>
+ <description>
+ Temporarily suspend fetching from a host if the
+ robots.txt response is HTTP 503 or any other 5xx server error
+ and HTTP 429 Too Many Requests. See also
+ http.robots.503.defer.visits.delay and
+ http.robots.503.defer.visits.retries
+ </description>
</property>
<property>
<name>http.robots.503.defer.visits.delay</name>
<value>300000</value>
- <description>Time in milliseconds to suspend crawling a host if the
- robots.txt response is HTTP 5xx or 429 Too Many Requests - see
- http.robots.503.defer.visits.</description>
+ <description>
+ Time in milliseconds to suspend crawling a host if the robots.txt
+ response is HTTP 5xx or 429 Too Many Requests - see
+ http.robots.503.defer.visits.
+ </description>
</property>
<property>
<name>http.robots.503.defer.visits.retries</name>
<value>3</value>
- <description>Number of retries crawling a host if the robots.txt
- response is HTTP 5xx or 429 - see http.robots.503.defer.visits.
- After n retries the host queue is dropped for this segment/cycle.
+ <description>
+ Number of retries crawling a host if the robots.txt response is
+ HTTP 5xx or 429 - see http.robots.503.defer.visits. After n
+ retries the host queue is dropped for this segment/cycle.
</description>
</property>
<property>
<name>http.robots.redirect.max</name>
<value>5</value>
- <description>Maximum number of redirects followed when fetching
- a robots.txt file. RFC 9309 specifies that "crawlers SHOULD
- follow at least five consecutive redirects, even across authorities
- (for example, hosts in the case of HTTP)."
+ <description>
+ Maximum number of redirects followed when fetching
+ a robots.txt file. RFC 9309 specifies that "crawlers SHOULD
+ follow at least five consecutive redirects, even across authorities
+ (for example, hosts in the case of HTTP)."
</description>
</property>
<property>
<name>http.agent.description</name>
<value></value>
- <description>Further description of our bot- this text is used in
- the User-Agent header. It appears in parenthesis after the agent name.
+ <description>
+ Further description of our bot- this text is used in
+ the User-Agent header. It appears in parenthesis after the agent name.
</description>
</property>
<property>
<name>http.agent.url</name>
<value></value>
- <description>A URL to advertise in the User-Agent header. This will
- appear in parenthesis after the agent name. Custom dictates that this
- should be a URL to a page that explains the purpose and behavior of this
- crawler.
+ <description>
+ A URL to advertise in the User-Agent header. This will appear in
+ parenthesis after the agent name. Custom dictates that this should
+ be a URL to a page that explains the purpose and behavior of this
+ crawler.
</description>
</property>
<property>
<name>http.agent.email</name>
<value></value>
- <description>An email address to advertise in the HTTP 'User-Agent' (and
+ <description>
+ An email address to advertise in the HTTP 'User-Agent' (and
'From') request headers. A good practice is to mangle this address
(e.g. 'info at example dot com') to avoid spamming.
</description>
@@ -204,8 +223,10 @@
<property>
<name>http.agent.version</name>
<value>Nutch-1.23-SNAPSHOT</value>
- <description>A version string to advertise in the User-Agent
- header.</description>
+ <description>
+ A version string to advertise in the User-Agent
+ header.
+ </description>
</property>
<property>
@@ -239,39 +260,44 @@
<property>
<name>http.agent.host</name>
<value></value>
- <description>Name or IP address of the host on which the Nutch crawler
- would be running. Currently this is used by 'protocol-httpclient'
- plugin.
+ <description>
+ Name or IP address of the host on which the Nutch crawler
+ would be running. Currently this is used by 'protocol-httpclient'
+ plugin.
</description>
</property>
<property>
<name>http.timeout</name>
<value>10000</value>
- <description>The default network timeout, in milliseconds.</description>
+ <description>
+ The default network timeout, in milliseconds.
+ </description>
</property>
<property>
<name>http.content.limit</name>
<value>1048576</value>
- <description>The length limit for downloaded content using the http/https
- protocols, in bytes. If this value is non-negative (>=0), content longer
- than it will be truncated; otherwise, no truncation at all. Do not
- confuse this setting with the file.content.limit setting.
+ <description>
+ The length limit for downloaded content using the http/https
+ protocols, in bytes. If this value is non-negative (>=0), content longer
+ than it will be truncated; otherwise, no truncation at all. Do not
+ confuse this setting with the file.content.limit setting.
</description>
</property>
<property>
<name>http.time.limit</name>
<value>-1</value>
- <description>The time limit in seconds to fetch a single document.
- If this value is non-negative (>=0), the HTTP protocol implementation
- will stop reading from a socket after http.time.limit seconds have
- been spent for fetching this document. The HTTP response is then
- marked as truncated. The http.time.limit should be set to a longer
- time period than http.timeout, as it applies to the entire duration
- to fetch a document, not only the network timeout of a single I/O
- operation. Note: supported only by protocol-okhttp.
+ <description>
+ The time limit in seconds to fetch a single document.
+ If this value is non-negative (>=0), the HTTP protocol implementation
+ will stop reading from a socket after http.time.limit seconds have
+ been spent for fetching this document. The HTTP response is then
+ marked as truncated. The http.time.limit should be set to a longer
+ time period than http.timeout, as it applies to the entire duration
+ to fetch a document, not only the network timeout of a single I/O
+ operation. Note: supported only by protocol-okhttp.
</description>
</property>
@@ -302,54 +328,62 @@
<property>
<name>http.proxy.host</name>
<value></value>
- <description>The proxy hostname. If empty, no proxy is used.</description>
+ <description>
+ The proxy hostname. If empty, no proxy is used.
+ </description>
</property>
<property>
<name>http.proxy.port</name>
<value></value>
- <description>The proxy port.</description>
+ <description>
+ The proxy port.
+ </description>
</property>
<property>
<name>http.proxy.username</name>
<value></value>
- <description>Username for proxy. This will be used by
- 'protocol-httpclient', if the proxy server requests basic, digest
- and/or NTLM authentication. To use this, 'protocol-httpclient' must
- be present in the value of 'plugin.includes' property.
- NOTE: For NTLM authentication, do not prefix the username with the
- domain, i.e. 'susam' is correct whereas 'DOMAIN\susam' is incorrect.
+ <description>
+ Username for proxy. This will be used by
+ 'protocol-httpclient', if the proxy server requests basic, digest
+ and/or NTLM authentication. To use this, 'protocol-httpclient' must
+ be present in the value of 'plugin.includes' property.
+ NOTE: For NTLM authentication, do not prefix the username with the
+ domain, i.e. 'susam' is correct whereas 'DOMAIN\susam' is incorrect.
</description>
</property>
<property>
<name>http.proxy.password</name>
<value></value>
- <description>Password for proxy. This will be used by
- 'protocol-httpclient', if the proxy server requests basic, digest
- and/or NTLM authentication. To use this, 'protocol-httpclient' must
- be present in the value of 'plugin.includes' property.
+ <description>
+ Password for proxy. This will be used by
+ 'protocol-httpclient', if the proxy server requests basic, digest
+ and/or NTLM authentication. To use this, 'protocol-httpclient' must
+ be present in the value of 'plugin.includes' property.
</description>
</property>
<property>
<name>http.proxy.realm</name>
<value></value>
- <description>Authentication realm for proxy. Do not define a value
- if realm is not required or authentication should take place for any
- realm. NTLM does not use the notion of realms. Specify the domain name
- of NTLM authentication as the value for this property. To use this,
- 'protocol-httpclient' must be present in the value of
- 'plugin.includes' property.
+ <description>
+ Authentication realm for proxy. Do not define a value
+ if realm is not required or authentication should take place for any
+ realm. NTLM does not use the notion of realms. Specify the domain name
+ of NTLM authentication as the value for this property. To use this,
+ 'protocol-httpclient' must be present in the value of
+ 'plugin.includes' property.
</description>
</property>
<property>
<name>http.auth.file</name>
<value>httpclient-auth.xml</value>
- <description>Authentication configuration file for
- 'protocol-httpclient' plugin.
+ <description>
+ Authentication configuration file for
+ 'protocol-httpclient' plugin.
</description>
</property>
@@ -365,7 +399,8 @@
<property>
<name>http.proxy.exception.list</name>
<value></value>
- <description>Either i) a comma separated list of hosts e.g.,
+ <description>
+ Either i) a comma separated list of hosts e.g.,
domain1.org,www.domain2.com, or ii) a wildcard '*' in either prefix
e.g. "*.domain.com", or suffix e.g. "some.domain.*", that don't
use the proxy (e.g. intranets)
@@ -396,57 +431,63 @@
<property>
<name>http.accept.language</name>
<value>en-us,en-gb,en;q=0.7,*;q=0.3</value>
- <description>Value of the "Accept-Language" request header field.
- This allows selecting non-English language as default one to retrieve.
- It is a useful setting for search engines build for certain national group.
- To send requests without "Accept-Language" header field, this property must
- be configured to contain a space character because an empty property does
- not overwrite the default.
+ <description>
+ Value of the "Accept-Language" request header field.
+ This allows selecting non-English language as default one to retrieve.
+ It is a useful setting for search engines build for certain national group.
+ To send requests without "Accept-Language" header field, this property must
+ be configured to contain a space character because an empty property does
+ not overwrite the default.
</description>
</property>
<property>
<name>http.accept</name>
<value>text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8</value>
- <description>Value of the "Accept" request header field. A space character
- as value will cause that no "Accept" header field is sent in the request.
+ <description>
+ Value of the "Accept" request header field. A space character
+ as value will cause that no "Accept" header field is sent in the request.
</description>
</property>
<property>
<name>http.accept.charset</name>
<value>utf-8,iso-8859-1;q=0.7,*;q=0.7</value>
- <description>Value of the "Accept-Charset" request header field. A space
character
- as value will cause that no "Accept-Charset" header field is sent in the
request.
+ <description>
+ Value of the "Accept-Charset" request header field. A space character
+ as value will cause that no "Accept-Charset" header field is sent in the
request.
</description>
</property>
<property>
<name>http.store.responsetime</name>
<value>true</value>
- <description>Enables us to record the response time of the
- host which is the time period between start connection to end
- connection of a pages host. The response time in milliseconds
- is stored in CrawlDb in CrawlDatum's meta data under key "_rs_"
+ <description>
+ Enables us to record the response time of the
+ host which is the time period between start connection to end
+ connection of a pages host. The response time in milliseconds
+ is stored in CrawlDb in CrawlDatum's meta data under key "_rs_"
</description>
</property>
<property>
<name>http.enable.if.modified.since.header</name>
<value>true</value>
- <description>Whether Nutch sends an HTTP If-Modified-Since header. It reduces
- bandwidth when enabled by not downloading pages that respond with an HTTP
- Not-Modified header. URLs that are not downloaded are not passed through
- parse or indexing filters. If you regularly modify filters, you should force
- Nutch to also download unmodified pages by disabling this feature.
+ <description>
+ Whether Nutch sends an HTTP If-Modified-Since header. It reduces
+ bandwidth when enabled by not downloading pages that respond with an HTTP
+ Not-Modified header. URLs that are not downloaded are not passed through
+ parse or indexing filters. If you regularly modify filters, you should
force
+ Nutch to also download unmodified pages by disabling this feature.
</description>
</property>
<property>
<name>http.enable.cookie.header</name>
<value>true</value>
- <description>Whether Nutch sends an HTTP Cookie header. The cookie value
- is read from the CrawlDatum Cookie metadata field.
+ <description>
+ Whether Nutch sends an HTTP Cookie header. The cookie value
+ is read from the CrawlDatum Cookie metadata field.
</description>
</property>
@@ -501,236 +542,272 @@
<property>
<name>ftp.username</name>
<value>anonymous</value>
- <description>ftp login username.</description>
+ <description>
+ ftp login username.
+ </description>
</property>
<property>
<name>ftp.password</name>
<value>[email protected]</value>
- <description>ftp login password.</description>
+ <description>
+ ftp login password.
+ </description>
</property>
<property>
<name>ftp.content.limit</name>
<value>1048576</value>
- <description>The length limit for downloaded content, in bytes.
- If this value is non-negative (>=0), content longer than it will be
truncated;
- otherwise, no truncation at all.
- Caution: classical ftp RFCs never defines partial transfer and, in fact,
- some ftp servers out there do not handle client side forced close-down very
- well. Our implementation tries its best to handle such situations smoothly.
+ <description>
+ The length limit for downloaded content, in bytes.
+ If this value is non-negative (>=0), content longer than it will be
truncated;
+ otherwise, no truncation at all.
+ Caution: classical ftp RFCs never defines partial transfer and, in fact,
+ some ftp servers out there do not handle client side forced close-down very
+ well. Our implementation tries its best to handle such situations smoothly.
</description>
</property>
<property>
<name>ftp.timeout</name>
<value>60000</value>
- <description>Default timeout for ftp client socket, in millisec.
- Please also see ftp.keep.connection below.</description>
+ <description>
+ Default timeout for ftp client socket, in millisec.
+ Please also see ftp.keep.connection below.
+ </description>
</property>
<property>
<name>ftp.server.timeout</name>
<value>100000</value>
- <description>An estimation of ftp server idle time, in millisec.
- Typically it is 120000 millisec for many ftp servers out there.
- Better be conservative here. Together with ftp.timeout, it is used to
- decide if we need to delete (annihilate) current ftp.client instance and
- force to start another ftp.client instance anew. This is necessary because
- a fetcher thread may not be able to obtain next request from queue in time
- (due to idleness) before our ftp client times out or remote server
- disconnects. Used only when ftp.keep.connection is true (please see below).
+ <description>
+ An estimation of ftp server idle time, in millisec.
+ Typically it is 120000 millisec for many ftp servers out there.
+ Better be conservative here. Together with ftp.timeout, it is used to
+ decide if we need to delete (annihilate) current ftp.client instance and
+ force to start another ftp.client instance anew. This is necessary because
+ a fetcher thread may not be able to obtain next request from queue in time
+ (due to idleness) before our ftp client times out or remote server
+ disconnects. Used only when ftp.keep.connection is true (please see below).
</description>
</property>
<property>
<name>ftp.keep.connection</name>
<value>false</value>
- <description>Whether to keep ftp connection. Useful if crawling same host
- again and again. When set to true, it avoids connection, login and dir list
- parser setup for subsequent URLs. If it is set to true, however, you must
- make sure (roughly):
- (1) ftp.timeout is less than ftp.server.timeout
- (2) ftp.timeout is larger than (fetcher.threads.fetch * fetcher.server.delay)
- Otherwise there will be too many "delete client because idled too long"
- messages in thread logs.</description>
+ <description>
+ Whether to keep ftp connection. Useful if crawling same host
+ again and again. When set to true, it avoids connection, login and dir list
+ parser setup for subsequent URLs. If it is set to true, however, you must
+ make sure (roughly):
+ (1) ftp.timeout is less than ftp.server.timeout
+ (2) ftp.timeout is larger than (fetcher.threads.fetch *
fetcher.server.delay)
+ Otherwise there will be too many "delete client because idled too long"
+ messages in thread logs.
+ </description>
</property>
<property>
<name>ftp.follow.talk</name>
<value>false</value>
- <description>Whether to log dialogue between our client and remote
- server. Useful for debugging.</description>
+ <description>
+ Whether to log dialogue between our client and remote
+ server. Useful for debugging.
+ </description>
</property>
<!-- web db properties -->
<property>
<name>db.fetch.interval.default</name>
<value>2592000</value>
- <description>The default number of seconds between re-fetches of a page (30
days).
+ <description>
+ The default number of seconds between re-fetches of a page (30 days).
</description>
</property>
<property>
<name>db.fetch.interval.max</name>
<value>7776000</value>
- <description>The maximum number of seconds between re-fetches of a page
- (90 days). After this period every page in the db will be re-tried, no
- matter what is its status.
+ <description>
+ The maximum number of seconds between re-fetches of a page
+ (90 days). After this period every page in the db will be re-tried, no
+ matter what is its status.
</description>
</property>
<property>
<name>db.fetch.schedule.class</name>
<value>org.apache.nutch.crawl.DefaultFetchSchedule</value>
- <description>The implementation of fetch schedule. DefaultFetchSchedule
simply
- adds the original fetchInterval to the last fetch time, regardless of
- page changes, whereas AdaptiveFetchSchedule (see below) tries to adapt
- to the rate at which a given page is changed.
+ <description>
+ The implementation of fetch schedule. DefaultFetchSchedule simply
+ adds the original fetchInterval to the last fetch time, regardless of
+ page changes, whereas AdaptiveFetchSchedule (see below) tries to adapt
+ to the rate at which a given page is changed.
</description>
</property>
<property>
<name>db.fetch.schedule.adaptive.inc_rate</name>
<value>0.4</value>
- <description>If a page is unmodified, its fetchInterval will be
- increased by this rate. This value should not
- exceed 0.5, otherwise the algorithm becomes unstable.</description>
+ <description>
+ If a page is unmodified, its fetchInterval will be
+ increased by this rate. This value should not
+ exceed 0.5, otherwise the algorithm becomes unstable.
+ </description>
</property>
<property>
<name>db.fetch.schedule.adaptive.dec_rate</name>
<value>0.2</value>
- <description>If a page is modified, its fetchInterval will be
- decreased by this rate. This value should not
- exceed 0.5, otherwise the algorithm becomes unstable.</description>
+ <description>
+ If a page is modified, its fetchInterval will be
+ decreased by this rate. This value should not
+ exceed 0.5, otherwise the algorithm becomes unstable.
+ </description>
</property>
<property>
<name>db.fetch.schedule.adaptive.min_interval</name>
<value>60.0</value>
- <description>Minimum fetchInterval, in seconds.</description>
+ <description>
+ Minimum fetchInterval, in seconds.
+ </description>
</property>
<property>
<name>db.fetch.schedule.adaptive.max_interval</name>
<value>31536000.0</value>
- <description>Maximum fetchInterval, in seconds (365 days).
- NOTE: this is limited by db.fetch.interval.max. Pages with
- fetchInterval larger than db.fetch.interval.max
- will be fetched anyway.</description>
+ <description>
+ Maximum fetchInterval, in seconds (365 days).
+ NOTE: this is limited by db.fetch.interval.max. Pages with
+ fetchInterval larger than db.fetch.interval.max
+ will be fetched anyway.
+ </description>
</property>
<property>
<name>db.fetch.schedule.adaptive.sync_delta</name>
<value>true</value>
- <description>If true, try to synchronize with the time of page change.
- by shifting the next fetchTime by a fraction (sync_rate) of the difference
- between the last modification time, and the last fetch time.</description>
+ <description>
+ If true, try to synchronize with the time of page change.
+ by shifting the next fetchTime by a fraction (sync_rate) of the difference
+ between the last modification time, and the last fetch time.
+ </description>
</property>
<property>
<name>db.fetch.schedule.adaptive.sync_delta_rate</name>
<value>0.3</value>
- <description>See sync_delta for description. This value should not
- exceed 0.5, otherwise the algorithm becomes unstable.</description>
+ <description>
+ See sync_delta for description. This value should not
+ exceed 0.5, otherwise the algorithm becomes unstable.
+ </description>
</property>
<property>
<name>db.fetch.schedule.mime.file</name>
<value>adaptive-mimetypes.txt</value>
- <description>The configuration file for the MimeAdaptiveFetchSchedule.
+ <description>
+ The configuration file for the MimeAdaptiveFetchSchedule.
</description>
</property>
<property>
<name>db.update.additions.allowed</name>
<value>true</value>
- <description>If true, updatedb will add newly discovered URLs, if false
- only already existing URLs in the CrawlDb will be updated and no new
- URLs will be added.
+ <description>
+ If true, updatedb will add newly discovered URLs, if false
+ only already existing URLs in the CrawlDb will be updated and no new
+ URLs will be added.
</description>
</property>
<property>
<name>db.preserve.backup</name>
<value>true</value>
- <description>If true, updatedb will keep a backup of the previous CrawlDb
- version in the old directory. In case of disaster, one can rename old to
- current and restore the CrawlDb to its previous state.
+ <description>
+ If true, updatedb will keep a backup of the previous CrawlDb
+ version in the old directory. In case of disaster, one can rename old to
+ current and restore the CrawlDb to its previous state.
</description>
</property>
<property>
<name>db.update.purge.404</name>
<value>false</value>
- <description>If true, updatedb will add purge records with status DB_GONE
- from the CrawlDb.
+ <description>
+ If true, updatedb will add purge records with status DB_GONE
+ from the CrawlDb.
</description>
</property>
<property>
<name>db.update.purge.orphans</name>
<value>false</value>
- <description>If true, updatedb will permanently delete URLs marked
- as orphan from the CrawlDb. The plugin scoring-orphan needs to be
- activated to get records marked as orphan. See the plugin's options
- elsewhere in this document.
+ <description>
+ If true, updatedb will permanently delete URLs marked
+ as orphan from the CrawlDb. The plugin scoring-orphan needs to be
+ activated to get records marked as orphan. See the plugin's options
+ elsewhere in this document.
</description>
</property>
<property>
- <name>crawldb.url.normalizers</name>
- <value>false</value>
- <description>
- !Temporary, can be overwritten with the command line!
- Normalize URLs when updating CrawlDb
- </description>
+ <name>crawldb.url.normalizers</name>
+ <value>false</value>
+ <description>
+ !Temporary, can be overwritten with the command line!
+ Normalize URLs when updating CrawlDb
+ </description>
</property>
<property>
- <name>crawldb.url.filters</name>
- <value>false</value>
- <description>
- !Temporary, can be overwritten with the command line!
- Filter URLs when updating CrawlDb
- </description>
+ <name>crawldb.url.filters</name>
+ <value>false</value>
+ <description>
+ !Temporary, can be overwritten with the command line!
+ Filter URLs when updating CrawlDb
+ </description>
</property>
<property>
<name>db.update.max.inlinks</name>
<value>10000</value>
- <description>Maximum number of inlinks to take into account when updating
- a URL score in the CrawlDb. Only the best scoring inlinks are kept.
+ <description>
+ Maximum number of inlinks to take into account when updating
+ a URL score in the CrawlDb. Only the best scoring inlinks are kept.
</description>
</property>
<property>
<name>db.ignore.internal.links</name>
<value>false</value>
- <description>If true, outlinks leading from a page to pages of the
- same host or domain will be ignored. See also
- 'db.ignore.external.links' and 'db.ignore.external.links.mode'.
+ <description>
+ If true, outlinks leading from a page to pages of the
+ same host or domain will be ignored. See also
+ 'db.ignore.external.links' and 'db.ignore.external.links.mode'.
</description>
</property>
<property>
<name>db.ignore.external.links</name>
<value>false</value>
- <description>If true, outlinks leading from a page to an external host or
domain
- will be ignored. This is an effective way to limit the crawl to include
- only initially injected hosts or domains, without creating complex
URLFilters.
- See also 'db.ignore.external.links.mode'.
+ <description>
+ If true, outlinks leading from a page to an external host or domain
+ will be ignored. This is an effective way to limit the crawl to include
+ only initially injected hosts or domains, without creating complex
URLFilters.
+ See also 'db.ignore.external.links.mode'.
</description>
</property>
<property>
<name>db.ignore.also.redirects</name>
<value>true</value>
- <description>If true, the fetcher checks redirects the same way as
- links when ignoring internal or external links. Set to false to
- follow redirects despite the values for db.ignore.external.links and
- db.ignore.internal.links.
+ <description>
+ If true, the fetcher checks redirects the same way as
+ links when ignoring internal or external links. Set to false to
+ follow redirects despite the values for db.ignore.external.links and
+ db.ignore.internal.links.
</description>
</property>
@@ -744,7 +821,7 @@
</description>
</property>
- <property>
+<property>
<name>db.ignore.external.exemptions.file</name>
<value>db-ignore-external-exemptions.txt</value>
<description>
@@ -755,61 +832,68 @@
<property>
<name>db.injector.overwrite</name>
<value>false</value>
- <description>Whether existing records in the CrawlDb will be overwritten
- by injected records.
+ <description>
+ Whether existing records in the CrawlDb will be overwritten
+ by injected records.
</description>
</property>
<property>
<name>db.injector.update</name>
<value>false</value>
- <description>If true existing records in the CrawlDb will be updated with
- injected records. Old meta data is preserved. The db.injector.overwrite
- parameter has precedence.
+ <description>
+ If true existing records in the CrawlDb will be updated with
+ injected records. Old meta data is preserved. The db.injector.overwrite
+ parameter has precedence.
</description>
</property>
<property>
<name>db.score.injected</name>
<value>1.0</value>
- <description>The score of new pages added by the injector.
+ <description>
+ The score of new pages added by the injector.
</description>
</property>
<property>
<name>db.score.link.external</name>
<value>1.0</value>
- <description>The score factor for new pages added due to a link from
- another host relative to the referencing page's score. Scoring plugins
- may use this value to affect initial scores of external links.
+ <description>
+ The score factor for new pages added due to a link from
+ another host relative to the referencing page's score. Scoring plugins
+ may use this value to affect initial scores of external links.
</description>
</property>
<property>
<name>db.score.link.internal</name>
<value>1.0</value>
- <description>The score factor for pages added due to a link from the
- same host, relative to the referencing page's score. Scoring plugins
- may use this value to affect initial scores of internal links.
+ <description>
+ The score factor for pages added due to a link from the
+ same host, relative to the referencing page's score. Scoring plugins
+ may use this value to affect initial scores of internal links.
</description>
</property>
<property>
<name>db.score.count.filtered</name>
<value>false</value>
- <description>The score value passed to newly discovered pages is
- calculated as a fraction of the original page score divided by the
- number of outlinks. If this option is false, only the outlinks that passed
- URLFilters will count, if it's true then all outlinks will count.
+ <description>
+ The score value passed to newly discovered pages is
+ calculated as a fraction of the original page score divided by the
+ number of outlinks. If this option is false, only the outlinks that passed
+ URLFilters will count, if it's true then all outlinks will count.
</description>
</property>
<property>
<name>db.max.outlinks.per.page</name>
<value>100</value>
- <description>The maximum number of outlinks that we'll process for a page.
- If this value is non-negative (>=0), at most db.max.outlinks.per.page
outlinks
- will be processed for a page; otherwise, all outlinks will be processed.
+ <description>
+ The maximum number of outlinks that we'll process for a page.
+ If this value is non-negative (>=0), at most db.max.outlinks.per.page
outlinks
+ will be processed for a page; otherwise, all outlinks will be processed.
</description>
</property>
@@ -841,32 +925,38 @@
<property>
<name>db.fetch.retry.max</name>
<value>3</value>
- <description>The maximum number of times a URL that has encountered
- recoverable errors is generated for fetch.</description>
+ <description>
+ The maximum number of times a URL that has encountered
+ recoverable errors is generated for fetch.
+ </description>
</property>
<property>
<name>db.signature.class</name>
<value>org.apache.nutch.crawl.MD5Signature</value>
- <description>The default implementation of a page signature. Signatures
- created with this implementation will be used for duplicate detection
- and removal.</description>
+ <description>
+ The default implementation of a page signature. Signatures
+ created with this implementation will be used for duplicate detection
+ and removal.
+ </description>
</property>
<property>
<name>db.signature.text_profile.min_token_len</name>
<value>2</value>
- <description>Minimum token length to be included in the signature.
+ <description>
+ Minimum token length to be included in the signature.
</description>
</property>
<property>
<name>db.signature.text_profile.quant_rate</name>
<value>0.01</value>
- <description>Profile frequencies will be rounded down to a multiple of
- QUANT = (int)(QUANT_RATE * maxFreq), where maxFreq is a maximum token
- frequency. If maxFreq > 1 then QUANT will be at least 2, which means that
- for longer texts tokens with frequency 1 will always be discarded.
+ <description>
+ Profile frequencies will be rounded down to a multiple of
+ QUANT = (int)(QUANT_RATE * maxFreq), where maxFreq is a maximum token
+ frequency. If maxFreq > 1 then QUANT will be at least 2, which means that
+ for longer texts tokens with frequency 1 will always be discarded.
</description>
</property>
@@ -885,28 +975,31 @@
<property>
<name>linkdb.max.inlinks</name>
<value>10000</value>
- <description>Maximum number of inlinks per URL to be kept in LinkDb.
- If "invertlinks" finds more inlinks than this number, only the first
- N inlinks will be stored, and the rest will be discarded.
+ <description>
+ Maximum number of inlinks per URL to be kept in LinkDb.
+ If "invertlinks" finds more inlinks than this number, only the first
+ N inlinks will be stored, and the rest will be discarded.
</description>
</property>
<property>
<name>linkdb.ignore.internal.links</name>
<value>true</value>
- <description>If true, when adding new links to a page, links from
- the same host are ignored. This is an effective way to limit the
- size of the link and anchor text database (LinkDb), keeping only the
- more descriptive external links and ignoring internal navigation
- links.
+ <description>
+ If true, when adding new links to a page, links from
+ the same host are ignored. This is an effective way to limit the
+ size of the link and anchor text database (LinkDb), keeping only the
+ more descriptive external links and ignoring internal navigation
+ links.
</description>
</property>
<property>
<name>linkdb.ignore.external.links</name>
<value>false</value>
- <description>If true, when adding new links to a page in the LinkDb,
- links from a different host are ignored.
+ <description>
+ If true, when adding new links to a page in the LinkDb,
+ links from a different host are ignored.
</description>
</property>
@@ -924,76 +1017,92 @@
<property>
<name>generate.max.count</name>
<value>-1</value>
- <description>The maximum number of URLs in a single
- fetchlist. -1 if unlimited. The URLs are counted according
- to the value of the parameter generate.count.mode.
+ <description>
+ The maximum number of URLs in a single
+ fetchlist. -1 if unlimited. The URLs are counted according
+ to the value of the parameter generate.count.mode.
</description>
</property>
<property>
<name>generate.count.mode</name>
<value>host</value>
- <description>Determines how the URLs are counted for generate.max.count.
- Default value is 'host' but can be 'domain'. Note that we do not count
- per IP in the new version of the Generator.
+ <description>
+ Determines how the URLs are counted for generate.max.count.
+ Default value is 'host' but can be 'domain'. Note that we do not count
+ per IP in the new version of the Generator.
</description>
</property>
<property>
<name>generate.update.crawldb</name>
<value>false</value>
- <description>For highly-concurrent environments, where several
- generate/fetch/update cycles may overlap, setting this to true ensures
- that generate will create different fetchlists even without intervening
- updatedb-s, at the cost of running an additional job to update CrawlDb.
- If false, running generate twice without intervening updatedb will
- generate identical fetchlists. See also crawl.gen.delay which defines
- how long items already generated are blocked.</description>
+ <description>
+ For highly-concurrent environments, where several
+ generate/fetch/update cycles may overlap, setting this to true ensures
+ that generate will create different fetchlists even without intervening
+ updatedb-s, at the cost of running an additional job to update CrawlDb.
+ If false, running generate twice without intervening updatedb will
+ generate identical fetchlists. See also crawl.gen.delay which defines
+ how long items already generated are blocked.
+ </description>
</property>
<property>
<name>generate.min.score</name>
<value>0</value>
- <description>Select only entries with a score larger than
- generate.min.score.</description>
+ <description>
+ Select only entries with a score larger than
+ generate.min.score.
+ </description>
</property>
<property>
<name>generate.min.interval</name>
<value>-1</value>
- <description>Select only entries with a retry interval lower than
- generate.min.interval. A value of -1 disables this check.</description>
+ <description>
+ Select only entries with a retry interval lower than
+ generate.min.interval. A value of -1 disables this check.
+ </description>
</property>
<property>
<name>generate.hostdb</name>
<value></value>
- <description>Path to HostDB, required for the generate.max.count.expr
- and generate.fetch.delay.expr properties.
- See https://issues.apache.org/jira/browse/NUTCH-2368</description>
+ <description>
+ Path to HostDB, required for the generate.max.count.expr
+ and generate.fetch.delay.expr properties.
+ See https://issues.apache.org/jira/browse/NUTCH-2368
+ </description>
</property>
<property>
<name>generate.fetch.delay.expr</name>
<value></value>
- <description>Controls variable fetcher.server.delay via a Jexl expression and
- HostDB information. It allows you to alter fetch delay based on HostDB data.
- See https://issues.apache.org/jira/browse/NUTCH-2368</description>
+ <description>
+ Controls variable fetcher.server.delay via a Jexl expression and
+ HostDB information. It allows you to alter fetch delay based on HostDB
data.
+ See https://issues.apache.org/jira/browse/NUTCH-2368
+ </description>
</property>
<property>
<name>generate.max.count.expr</name>
<value></value>
- <description>Controls variable generate.max.count via a Jexl expression and
- HostDB information. It allows you to alter maxCount based on HostDB data.
- See https://issues.apache.org/jira/browse/NUTCH-2368</description>
+ <description>
+ Controls variable generate.max.count via a Jexl expression and
+ HostDB information. It allows you to alter maxCount based on HostDB data.
+ See https://issues.apache.org/jira/browse/NUTCH-2368
+ </description>
</property>
<property>
<name>generate.restrict.status</name>
<value></value>
- <description>Select only entries of this status, see
- https://issues.apache.org/jira/browse/NUTCH-1248</description>
+ <description>
+ Select only entries of this status, see
+ https://issues.apache.org/jira/browse/NUTCH-1248
+ </description>
</property>
<!-- urlpartitioner properties -->
@@ -1001,8 +1110,9 @@
<property>
<name>partition.url.mode</name>
<value>byHost</value>
- <description>Determines how to partition URLs. Default value is 'byHost',
- also takes 'byDomain' or 'byIP'.
+ <description>
+ Determines how to partition URLs. Default value is 'byHost',
+ also takes 'byDomain' or 'byIP'.
</description>
</property>
@@ -1023,324 +1133,365 @@
<property>
<name>fetcher.server.delay</name>
<value>5.0</value>
- <description>The number of seconds the fetcher will delay between
+ <description>
+ The number of seconds the fetcher will delay between
successive requests to the same server. Note that this might get
overridden by a Crawl-Delay from a robots.txt and is used ONLY if
fetcher.threads.per.queue is set to 1.
- </description>
+ </description>
</property>
<property>
<name>fetcher.server.min.delay</name>
<value>0.0</value>
- <description>The minimum number of seconds the fetcher will delay between
- successive requests to the same server. This value is applicable ONLY
- if fetcher.threads.per.queue is greater than 1 (i.e. the host blocking
- is turned off).</description>
+ <description>
+ The minimum number of seconds the fetcher will delay between
+ successive requests to the same server. This value is applicable ONLY
+ if fetcher.threads.per.queue is greater than 1 (i.e. the host blocking
+ is turned off).
+ </description>
</property>
<property>
- <name>fetcher.max.crawl.delay</name>
- <value>30</value>
- <description>
- If the Crawl-Delay in robots.txt is set to greater than this value (in
- seconds) then the fetcher will skip this page, generating an error report.
- If set to -1 the fetcher will never skip such pages and will wait the
- amount of time retrieved from robots.txt Crawl-Delay, however long that
- might be.
- </description>
+ <name>fetcher.max.crawl.delay</name>
+ <value>30</value>
+ <description>
+ If the Crawl-Delay in robots.txt is set to greater than this value (in
+ seconds) then the fetcher will skip this page, generating an error report.
+ If set to -1 the fetcher will never skip such pages and will wait the
+ amount of time retrieved from robots.txt Crawl-Delay, however long that
+ might be.
+ </description>
</property>
<property>
- <name>fetcher.min.crawl.delay</name>
- <value>${fetcher.server.delay}</value>
- <description>
- Minimum Crawl-Delay (in seconds) accepted in robots.txt, even if the
- robots.txt specifies a shorter delay. By default the minimum Crawl-Delay
- is set to the value of `fetcher.server.delay` which guarantees that
- a value set in the robots.txt cannot make the crawler more aggressive
- than the default configuration.
- </description>
+ <name>fetcher.min.crawl.delay</name>
+ <value>${fetcher.server.delay}</value>
+ <description>
+ Minimum Crawl-Delay (in seconds) accepted in robots.txt, even if the
+ robots.txt specifies a shorter delay. By default the minimum Crawl-Delay
+ is set to the value of `fetcher.server.delay` which guarantees that
+ a value set in the robots.txt cannot make the crawler more aggressive
+ than the default configuration.
+ </description>
</property>
<property>
<name>fetcher.threads.fetch</name>
<value>10</value>
- <description>The number of FetcherThreads the fetcher should use.
- This is also determines the maximum number of requests that are
- made at once (each FetcherThread handles one connection). The total
- number of threads running in distributed mode will be the number of
- fetcher threads * number of nodes as fetcher has one map task per node.
+ <description>
+ The number of FetcherThreads the fetcher should use.
+ This is also determines the maximum number of requests that are
+ made at once (each FetcherThread handles one connection). The total
+ number of threads running in distributed mode will be the number of
+ fetcher threads * number of nodes as fetcher has one map task per node.
</description>
</property>
<property>
<name>fetcher.threads.start.delay</name>
<value>10</value>
- <description>Delay in milliseconds between starting Fetcher threads
- to avoid that all threads simultaneously fetch the first pages and
- cause that DNS or other resources are temporarily exhausted.
+ <description>
+ Delay in milliseconds between starting Fetcher threads
+ to avoid that all threads simultaneously fetch the first pages and
+ cause that DNS or other resources are temporarily exhausted.
</description>
</property>
<property>
<name>fetcher.threads.per.queue</name>
<value>1</value>
- <description>This number is the maximum number of threads that
+ <description>
+ This number is the maximum number of threads that
should be allowed to access a queue at one time. Setting it to
a value > 1 will cause the Crawl-Delay value from robots.txt to
be ignored and the value of fetcher.server.min.delay to be used
as a delay between successive requests to the same server instead
of fetcher.server.delay.
- </description>
+ </description>
</property>
<property>
<name>fetcher.queue.mode</name>
<value>byHost</value>
- <description>Determines how to put URLs into queues. Default value
- is 'byHost', also takes 'byDomain' or 'byIP'. Crawl delays are
- implemented on the level of fetcher queues.
+ <description>
+ Determines how to put URLs into queues. Default value
+ is 'byHost', also takes 'byDomain' or 'byIP'. Crawl delays are
+ implemented on the level of fetcher queues.
</description>
</property>
<property>
<name>http.log.exceptions.suppress.stack</name>
<value>java.net.UnknownHostException,java.net.NoRouteToHostException</value>
- <description>Comma-separated list of exceptions not shown with full
- stack trace in logs of fetcher and HTTP protocol implementations.
- The logs may shrink in size significantly, e.g., when for a large
- unrestricted web crawl unknown hosts are logged shortly without full
- stack trace. The full class name of the exception class (extending
- Throwable) including the package path must be specified.</description>
+ <description>
+ Comma-separated list of exceptions not shown with full
+ stack trace in logs of fetcher and HTTP protocol implementations.
+ The logs may shrink in size significantly, e.g., when for a large
+ unrestricted web crawl unknown hosts are logged shortly without full
+ stack trace. The full class name of the exception class (extending
+ Throwable) including the package path must be specified.
+ </description>
</property>
<property>
<name>fetcher.parse</name>
<value>false</value>
- <description>If true, fetcher will parse content. Default is false, which
means
- that a separate parsing step is required after fetching is
finished.</description>
+ <description>
+ If true, fetcher will parse content. Default is false, which means
+ that a separate parsing step is required after fetching is finished.
+ </description>
</property>
<property>
<name>fetcher.store.content</name>
<value>true</value>
- <description>If true, fetcher will store content.</description>
+ <description>
+ If true, fetcher will store content.
+ </description>
</property>
<property>
<name>fetcher.signature</name>
<value>false</value>
- <description>If true, fetcher will generate the signature for
- successfully fetched documents even if the content is not parsed by
- fetcher (see property fetcher.parse). Default is false, which means
- that the signature is calculated when parsing either by the fetcher
- or during the parsing step. Note that a non-parsing fetcher can
- only generate signatures based on the binary content and not on the
- textual content. An appropriate signature class should be chosen
- (see property db.signature.class).
+ <description>
+ If true, fetcher will generate the signature for
+ successfully fetched documents even if the content is not parsed by
+ fetcher (see property fetcher.parse). Default is false, which means
+ that the signature is calculated when parsing either by the fetcher
+ or during the parsing step. Note that a non-parsing fetcher can
+ only generate signatures based on the binary content and not on the
+ textual content. An appropriate signature class should be chosen
+ (see property db.signature.class).
</description>
</property>
<property>
<name>fetcher.timelimit.mins</name>
<value>-1</value>
- <description>This is the number of minutes allocated to the fetching.
- Once this value is reached, any remaining entry from the input URL list is
skipped
- and all active queues are emptied. The default value of -1 deactivates the
time limit.
+ <description>
+ This is the number of minutes allocated to the fetching.
+ Once this value is reached, any remaining entry from the input URL list is
skipped
+ and all active queues are emptied. The default value of -1 deactivates the
time limit.
</description>
</property>
<property>
<name>fetcher.max.exceptions.per.queue</name>
<value>5</value>
- <description>The maximum number of protocol-level exceptions
- (e.g. timeouts) or HTTP status codes mapped to ProtocolStatus.EXCEPTION
- per host (or IP) queue. Once this value is reached, any remaining entries
- from this queue are purged, effectively stopping the fetching from this
- host/IP. A value of -1 deactivates this limit.
+ <description>
+ The maximum number of protocol-level exceptions
+ (e.g. timeouts) or HTTP status codes mapped to ProtocolStatus.EXCEPTION
+ per host (or IP) queue. Once this value is reached, any remaining entries
+ from this queue are purged, effectively stopping the fetching from this
+ host/IP. A value of -1 deactivates this limit.
- Note that the exponential backoff mechanism (see the property
- fetcher.exceptions.per.queue.delay) causes increasing wait times
- after each exception in a queue. If there is no time limit
- (fetcher.timelimit.mins) or minimum throughput
- (fetcher.throughput.threshold.pages) configured, it is recommended
- to set this property to a considerably low value. This avoids the
- fetch process from hanging when only URLs in blocked queues remain.
+ Note that the exponential backoff mechanism (see the property
+ fetcher.exceptions.per.queue.delay) causes increasing wait times
+ after each exception in a queue. If there is no time limit
+ (fetcher.timelimit.mins) or minimum throughput
+ (fetcher.throughput.threshold.pages) configured, it is recommended
+ to set this property to a considerably low value. This avoids the
+ fetch process from hanging when only URLs in blocked queues remain.
</description>
</property>
<property>
<name>fetcher.exceptions.per.queue.delay</name>
<value>${fetcher.server.delay}</value>
- <description>Initial value (in seconds) of an additional dynamic
- delay slowing down fetches from a queue after an exception has
- occurred (see also fetcher.max.exceptions.per.queue). Starting with
- the initial value the delay doubles with every observed exception:
+ <description>
+ Initial value (in seconds) of an additional dynamic
+ delay slowing down fetches from a queue after an exception has
+ occurred (see also fetcher.max.exceptions.per.queue). Starting with
+ the initial value the delay doubles with every observed exception:
delay = fetcher.exceptions.per.queue.delay * 2^num_exception_in_queue
- An initial value of 0.0 disables this exponential backoff mechanism.
+ An initial value of 0.0 disables this exponential backoff mechanism.
</description>
</property>
<property>
<name>fetcher.exceptions.per.queue.clear.after</name>
<value>1800</value>
- <description>Time in seconds after which exception counters in
- queues can be cleared. This happens only if no items are queued in
- this queue and the configured time span has elapsed in addition to
- the crawl delay including the exponential backoff time, see
- fetcher.exceptions.per.queue.delay.
+ <description>
+ Time in seconds after which exception counters in
+ queues can be cleared. This happens only if no items are queued in
+ this queue and the configured time span has elapsed in addition to
+ the crawl delay including the exponential backoff time, see
+ fetcher.exceptions.per.queue.delay.
</description>
</property>
<property>
<name>fetcher.throughput.threshold.pages</name>
<value>-1</value>
- <description>The threshold of minimum pages per second. If the fetcher
downloads less
- pages per second than the configured threshold, the fetcher stops,
preventing slow queue's
- from stalling the throughput. This threshold must be an integer. This can be
useful when
- fetcher.timelimit.mins is hard to determine. The default value of -1
disables this check.
+ <description>
+ The threshold of minimum pages per second. If the fetcher downloads less
+ pages per second than the configured threshold, the fetcher stops,
preventing slow queue's
+ from stalling the throughput. This threshold must be an integer. This can
be useful when
+ fetcher.timelimit.mins is hard to determine. The default value of -1
disables this check.
</description>
</property>
<property>
<name>fetcher.throughput.threshold.retries</name>
<value>5</value>
- <description>The number of times the fetcher.throughput.threshold.pages is
allowed to be exceeded.
- This settings prevents accidental slow downs from immediately shutting down
the fetcher threads.
- The throughput is checked approx. every second. Note: the number of retries
should be lower
- than the timeout defined by mapreduce.task.timeout and
fetcher.threads.timeout.divisor (see there).
- For the default values, the timeout is 300 seconds. Consequently, the number
of retries should be
- significantly lower.
+ <description>
+ The number of times the fetcher.throughput.threshold.pages is allowed to
be exceeded.
+ This settings prevents accidental slow downs from immediately shutting
down the fetcher threads.
+ The throughput is checked approx. every second. Note: the number of
retries should be lower
+ than the timeout defined by mapreduce.task.timeout and
fetcher.threads.timeout.divisor (see there).
+ For the default values, the timeout is 300 seconds. Consequently, the
number of retries should be
+ significantly lower.
</description>
</property>
<property>
<name>fetcher.throughput.threshold.check.after</name>
<value>5</value>
- <description>The number of minutes after which the throughput check is
enabled.</description>
+ <description>
+ The number of minutes after which the throughput check is enabled.
+ </description>
</property>
<property>
<name>fetcher.threads.timeout.divisor</name>
<value>2</value>
- <description>(EXPERT) The thread time-out divisor to use. By default threads
have a time-out
- value of mapreduce.task.timeout / 2. Increase this setting if the fetcher
waits too
- long before killing hung threads. Be careful, a too high setting (+8) will
most likely kill the
- fetcher threads prematurely. The fetcher thread time-out avoids that the
task timeout (defined by
- the Hadoop configuration property mapreduce.task.timeout) is reached and the
fetcher job is failed.
+ <description>
+ (EXPERT) The thread time-out divisor to use. By default threads have a
time-out
+ value of mapreduce.task.timeout / 2. Increase this setting if the fetcher
waits too
+ long before killing hung threads. Be careful, a too high setting (+8) will
most likely kill the
+ fetcher threads prematurely. The fetcher thread time-out avoids that the
task timeout (defined by
+ the Hadoop configuration property mapreduce.task.timeout) is reached and
the fetcher job is failed.
</description>
</property>
<property>
<name>fetcher.queue.depth.multiplier</name>
<value>50</value>
- <description>(EXPERT)The fetcher buffers the incoming URLs into queues based
on the [host|domain|IP]
- (see param fetcher.queue.mode). The depth of the queue is the number of
threads times the value of this parameter.
- A large value requires more memory but can improve the performance of the
fetch when the order of the URLs in the fetch list
- is not optimal.
+ <description>
+ (EXPERT) The fetcher buffers the incoming URLs into queues based on the
[host|domain|IP]
+ (see param fetcher.queue.mode). The depth of the queue is the number of
threads times the value of this parameter.
+ A large value requires more memory, but can improve the performance of the
fetch when the order of the URLs
+ in the fetch list is not optimal.
</description>
</property>
<property>
<name>fetcher.follow.outlinks.depth</name>
<value>-1</value>
- <description>(EXPERT)When fetcher.parse is true and this value is greater
than 0 the fetcher will extract outlinks
- and follow until the desired depth is reached. A value of 1 means all
generated pages are fetched and their first degree
- outlinks are fetched and parsed too. Be careful, this feature is in itself
agnostic of the state of the CrawlDb and does not
- know about already fetched pages. A setting larger than 2 will most likely
fetch home pages twice in the same fetch cycle.
- It is highly recommended to set db.ignore.external.links to true to restrict
the outlink follower to URLs within the same
- domain. When disabled (false) the feature is likely to follow duplicates
even when depth=1.
- A value of -1 of 0 disables this feature.
+ <description>
+ (EXPERT) When fetcher.parse is true and this value is greater than 0 the
fetcher will extract outlinks
+ and follow until the desired depth is reached. A value of 1 means all
generated pages are fetched and their first degree
+ outlinks are fetched and parsed too. Be careful, this feature is in itself
agnostic of the state of the CrawlDb and does not
+ know about already fetched pages. A setting larger than 2 will most likely
fetch home pages twice in the same fetch cycle.
+ It is highly recommended to set db.ignore.external.links to true to
restrict the outlink follower to URLs within the same
+ domain. When disabled (false) the feature is likely to follow duplicates
even when depth=1.
+ A value of -1 of 0 disables this feature.
</description>
</property>
<property>
<name>fetcher.follow.outlinks.num.links</name>
<value>4</value>
- <description>(EXPERT)The number of outlinks to follow when
fetcher.follow.outlinks.depth is enabled. Be careful, this can multiply
- the total number of pages to fetch. This works with
fetcher.follow.outlinks.depth.divisor, by default settings the followed outlinks
- at depth 1 is 8, not 4.
+ <description>
+ (EXPERT) The number of outlinks to follow when
fetcher.follow.outlinks.depth is enabled. Be careful, this can multiply
+ the total number of pages to fetch. This works with
fetcher.follow.outlinks.depth.divisor, by default settings the followed outlinks
+ at depth 1 is 8, not 4.
</description>
</property>
<property>
<name>fetcher.follow.outlinks.depth.divisor</name>
<value>2</value>
- <description>(EXPERT)The divisor of fetcher.follow.outlinks.num.links per
fetcher.follow.outlinks.depth. This decreases the number
- of outlinks to follow by increasing depth. The formula used is: outlinks =
floor(divisor / depth * num.links). This prevents
- exponential growth of the fetch list.
+ <description>
+ (EXPERT) The divisor of fetcher.follow.outlinks.num.links per
fetcher.follow.outlinks.depth. This decreases the number
+ of outlinks to follow by increasing depth. The formula used is: outlinks =
floor(divisor / depth * num.links). This prevents
+ exponential growth of the fetch list.
</description>
</property>
<property>
<name>fetcher.follow.outlinks.ignore.external</name>
<value>true</value>
- <description>Whether to ignore or follow external links. Set
db.ignore.external.links to false and this to true to store outlinks
- in the output but not follow them. If db.ignore.external.links is true this
directive is ignored.
+ <description>
+ Whether to ignore or follow external links. Set db.ignore.external.links
to false and this to true to store outlinks
+ in the output but not follow them. If db.ignore.external.links is true
this directive is ignored.
</description>
</property>
<property>
<name>fetcher.bandwidth.target</name>
<value>-1</value>
- <description>Target bandwidth in kilobits per sec for each mapper instance.
This is used to adjust the number of
- fetching threads automatically (up to fetcher.maxNum.threads). A value of -1
deactivates the functionality, in which case
- the number of fetching threads is fixed (see
fetcher.threads.fetch).</description>
+ <description>
+ Target bandwidth in kilobits per sec for each mapper instance. This is
used to adjust the number of
+ fetching threads automatically (up to fetcher.maxNum.threads). A value of
-1 deactivates the functionality, in which case
+ the number of fetching threads is fixed (see fetcher.threads.fetch).
+ </description>
</property>
<property>
<name>fetcher.maxNum.threads</name>
<value>25</value>
- <description>Max number of fetch threads allowed when using
fetcher.bandwidth.target. Defaults to fetcher.threads.fetch if unspecified or
- set to a value lower than it. </description>
+ <description>
+ Max number of fetch threads allowed when using fetcher.bandwidth.target.
Defaults to fetcher.threads.fetch if unspecified or
+ set to a value lower than it. </description>
</property>
<property>
<name>fetcher.bandwidth.target.check.everyNSecs</name>
<value>30</value>
- <description>(EXPERT) Value in seconds which determines how frequently we
should reassess the optimal number of fetch threads when using
- fetcher.bandwidth.target. Defaults to 30 and must be at least
1.</description>
+ <description>
+ (EXPERT) Value in seconds which determines how frequently we should
reassess the optimal number of fetch threads when using
+ fetcher.bandwidth.target. Defaults to 30 and must be at least 1.
+ </description>
</property>
<property>
<name>fetcher.store.robotstxt</name>
<value>false</value>
- <description>If true (and fetcher.store.content is also true),
- fetcher will store the robots.txt response content and status for
- debugging or archival purposes. The robots.txt is added to the
- content/ folder of the fetched segment.
+ <description>
+ If true (and fetcher.store.content is also true),
+ fetcher will store the robots.txt response content and status for
+ debugging or archival purposes. The robots.txt is added to the
+ content/ folder of the fetched segment.
</description>
</property>
<property>
- <name>fetcher.publisher</name>
- <value>false</value>
- <description>Set this value to true if you want to use an
implementation of the Publisher/Subscriber model. Make sure to set corresponding
- Publisher implementation specific properties</description>
+ <name>fetcher.publisher</name>
+ <value>false</value>
+ <description>
+ Set this value to true if you want to use an implementation of the
Publisher/Subscriber model. Make sure to set corresponding
+ Publisher implementation specific properties
+ </description>
</property>
<property>
<name>fetcher.filter.urls</name>
<value>false</value>
- <description>Whether fetcher will filter URLs (with the configured URL
filters).</description>
+ <description>
+ Whether fetcher will filter URLs (with the configured URL filters).
+ </description>
</property>
<property>
<name>fetcher.normalize.urls</name>
<value>false</value>
- <description>Whether fetcher will normalize URLs (with the configured URL
normalizers).</description>
+ <description>
+ Whether fetcher will normalize URLs (with the configured URL normalizers).
+ </description>
</property>
<property>
<name>http.redirect.max</name>
<value>0</value>
- <description>The maximum number of redirects the fetcher will follow when
- trying to fetch a page. If set to negative or 0, fetcher won't immediately
- follow redirected URLs, instead it will record them for later fetching.
+ <description>
+ The maximum number of redirects the fetcher will follow when
+ trying to fetch a page. If set to negative or 0, fetcher won't immediately
+ follow redirected URLs, instead it will record them for later fetching.
</description>
</property>
@@ -1396,28 +1547,31 @@
<property>
<name>moreIndexingFilter.indexMimeTypeParts</name>
<value>true</value>
- <description>Determines whether the index-more plugin will split the
mime-type
- in sub parts, this requires the type field to be multi valued. Set to true
for backward
- compatibility. False will not split the mime-type.
+ <description>
+ Determines whether the index-more plugin will split the mime-type
+ in sub parts, this requires the type field to be multi valued. Set to true
for backward
+ compatibility. False will not split the mime-type.
</description>
</property>
<property>
<name>moreIndexingFilter.mapMimeTypes</name>
<value>false</value>
- <description>Determines whether MIME-type mapping is enabled. It takes a
- plain text file with mapped MIME-types. With it the user can map both
- application/xhtml+xml and text/html to the same target MIME-type so it
- can be treated equally in an index. See conf/contenttype-mapping.txt.
+ <description>
+ Determines whether MIME-type mapping is enabled. It takes a
+ plain text file with mapped MIME-types. With it the user can map both
+ application/xhtml+xml and text/html to the same target MIME-type so it
+ can be treated equally in an index. See conf/contenttype-mapping.txt.
</description>
</property>
<property>
<name>moreIndexingFilter.mapMimeTypes.field</name>
<value></value>
- <description>It's used if moreIndexingFilter.mapMimeTypes is true. Indicates
the field
- where the mapped MIME-type must be written. If it's empty or unset, the
content of the field "type"
- will be replaced by the mapped MIME-type.
+ <description>
+ It's used if moreIndexingFilter.mapMimeTypes is true. Indicates the field
+ where the mapped MIME-type must be written. If it's empty or unset, the
content of the field "type"
+ will be replaced by the mapped MIME-type.
</description>
</property>
@@ -1426,9 +1580,10 @@
<property>
<name>anchorIndexingFilter.deduplicate</name>
<value>false</value>
- <description>With this enabled the indexer will case-insensitive deduplicate
anchors
- before indexing. This prevents possible hundreds or thousands of identical
anchors for
- a given page to be indexed but will affect the search scoring (i.e. tf=1.0f).
+ <description>
+ With this enabled the indexer will case-insensitive deduplicate anchors
+ before indexing. This prevents possible hundreds or thousands of identical
anchors for
+ a given page to be indexed but will affect the search scoring (i.e.
tf=1.0f).
</description>
</property>
@@ -1437,84 +1592,97 @@
<property>
<name>indexingfilter.order</name>
<value></value>
- <description>The order by which index filters are applied.
- If empty, all available index filters (as dictated by properties
- plugin-includes and plugin-excludes above) are loaded and applied in system
- defined order. If not empty, only named filters are loaded and applied
- in given order. For example, if this property has value:
- org.apache.nutch.indexer.basic.BasicIndexingFilter
org.apache.nutch.indexer.more.MoreIndexingFilter
- then BasicIndexingFilter is applied first, and MoreIndexingFilter second.
+ <description>
+ The order by which index filters are applied.
+ If empty, all available index filters (as dictated by properties
+ plugin-includes and plugin-excludes above) are loaded and applied in system
+ defined order. If not empty, only named filters are loaded and applied
+ in given order. For example, if this property has value:
+ org.apache.nutch.indexer.basic.BasicIndexingFilter
org.apache.nutch.indexer.more.MoreIndexingFilter
+ then BasicIndexingFilter is applied first, and MoreIndexingFilter second.
- Filter ordering might have impact on result if one filter depends on output
of
- another filter.
+ Filter ordering might have impact on result if one filter depends on
output of
+ another filter.
</description>
</property>
<property>
<name>indexer.score.power</name>
<value>0.5</value>
- <description>Determines the power of link analyis scores. The boost
- of each page is set to <i>score<sup>scorePower</sup></i> where
- <i>score</i> is its link analysis score and <i>scorePower</i> is the
- value of this parameter. This is compiled into indexes, so, when
- this is changed, pages must be re-indexed for it to take
- effect.</description>
+ <description>
+ Determines the power of link analyis scores. The boost
+ of each page is set to <i>score<sup>scorePower</sup></i> where
+ <i>score</i> is its link analysis score and <i>scorePower</i> is the
+ value of this parameter. This is compiled into indexes, so, when
+ this is changed, pages must be re-indexed for it to take
+ effect.
+ </description>
</property>
<property>
<name>indexer.max.title.length</name>
<value>100</value>
- <description>The maximum number of characters of a title that are indexed. A
value of -1 disables this check.
+ <description>
+ The maximum number of characters of a title that are indexed. A value of
-1 disables this check.
</description>
</property>
<property>
<name>indexer.max.content.length</name>
<value>-1</value>
- <description>The maximum number of characters of a content that are indexed.
- Content beyond the limit is truncated. A value of -1 disables this check.
+ <description>
+ The maximum number of characters of a content that are indexed.
+ Content beyond the limit is truncated. A value of -1 disables this check.
</description>
</property>
<property>
<name>indexer.add.domain</name>
<value>false</value>
- <description>Whether to add the domain field to a
NutchDocument.</description>
+ <description>
+ Whether to add the domain field to a NutchDocument.
+ </description>
</property>
<property>
<name>indexer.skip.notmodified</name>
<value>false</value>
- <description>Whether the indexer will skip records with a db_notmodified
status.
+ <description>
+ Whether the indexer will skip records with a db_notmodified status.
</description>
</property>
<property>
<name>indexer.delete.robots.noindex</name>
<value>false</value>
- <description>Whether the indexer will delete documents marked by
robots=noindex
+ <description>
+ Whether the indexer will delete documents marked by robots=noindex
</description>
</property>
<property>
<name>indexer.delete.skipped.by.indexingfilter</name>
<value>false</value>
- <description>Whether the indexer will delete documents that were skipped by
indexing filters
+ <description>
+ Whether the indexer will delete documents that were skipped by indexing
filters
</description>
</property>
<property>
<name>indexer.delete</name>
<value>false</value>
- <description>Whether the indexer will delete documents that are gone. Gone
pages include redirects and duplicates.
- See also: 'link.delete.gone'.
+ <description>
+ Whether the indexer will delete documents that are gone. Gone pages
include redirects and duplicates.
+ See also: 'link.delete.gone'.
</description>
</property>
<property>
<name>indexer.indexwriters.file</name>
<value>index-writers.xml</value>
- <description>The configuration file for index writers.</description>
+ <description>
+ The configuration file for index writers.
+ </description>
</property>
<!-- Exchanges properties -->
@@ -1522,7 +1690,9 @@
<property>
<name>exchanges.exchanges.file</name>
<value>exchanges.xml</value>
- <description>The configuration file used by the Exchange
component.</description>
+ <description>
+ The configuration file used by the Exchange component.
+ </description>
</property>
<!-- URL normalizer properties -->
@@ -1530,71 +1700,81 @@
<property>
<name>urlnormalizer.order</name>
<value>org.apache.nutch.net.urlnormalizer.basic.BasicURLNormalizer
org.apache.nutch.net.urlnormalizer.regex.RegexURLNormalizer</value>
- <description>Order in which normalizers will run. If any of these isn't
- activated it will be silently skipped. If other normalizers not on the
- list are activated, they will run in random order after the ones
- specified here are run.
+ <description>
+ Order in which normalizers will run. If any of these isn't
+ activated it will be silently skipped. If other normalizers not on the
+ list are activated, they will run in random order after the ones
+ specified here are run.
</description>
</property>
<property>
<name>urlnormalizer.regex.file</name>
<value>regex-normalize.xml</value>
- <description>Name of the config file used by the RegexUrlNormalizer class.
+ <description>
+ Name of the config file used by the RegexUrlNormalizer class.
</description>
</property>
<property>
<name>urlnormalizer.loop.count</name>
<value>1</value>
- <description>Optionally loop through normalizers several times, to make
- sure that all transformations have been performed.
+ <description>
+ Optionally loop through normalizers several times, to make
+ sure that all transformations have been performed.
</description>
</property>
<property>
<name>urlnormalizer.basic.host.idn</name>
<value></value>
- <description>Let urlnormalizer-basic
- (org.apache.nutch.net.urlnormalizer.basic.BasicURLNormalizer)
- normalize Internationalized Domain Names (IDNs). Possible values
- are: `toAscii` - convert the Unicode form to the ASCII (Punycode)
- representation, `toUnicode` - convert ASCII (Punycode) to Unicode,
- or if left empty no normalization of IDNs is performed.
+ <description>
+ Let urlnormalizer-basic
+ (org.apache.nutch.net.urlnormalizer.basic.BasicURLNormalizer)
+ normalize Internationalized Domain Names (IDNs). Possible values
+ are: `toAscii` - convert the Unicode form to the ASCII (Punycode)
+ representation, `toUnicode` - convert ASCII (Punycode) to Unicode,
+ or if left empty no normalization of IDNs is performed.
</description>
</property>
<property>
<name>urlnormalizer.basic.host.idna2008</name>
<value>false</value>
- <description>If true, let urlnormalizer-basic
- normalize Internationalized Domain Names (IDNs) using the
- standard IDNA2008 (RFC 5890). If false, use IDNA2003 (RFC 3490).
- Note that urlnormalizer.basic.host.idn must be set, otherwise
- this property has no effect.
+ <description>
+ If true, let urlnormalizer-basic
+ normalize Internationalized Domain Names (IDNs) using the
+ standard IDNA2008 (RFC 5890). If false, use IDNA2003 (RFC 3490).
+ Note that urlnormalizer.basic.host.idn must be set, otherwise
+ this property has no effect.
</description>
</property>
<property>
<name>urlnormalizer.basic.host.trim-trailing-dot</name>
<value>false</value>
- <description>urlnormalizer-basic: Trim a trailing dot in host names:
- `https://example.org./` is normalized to `https://example.org/`.
+ <description>
+ urlnormalizer-basic: Trim a trailing dot in host names:
+ `https://example.org./` is normalized to `https://example.org/`.
</description>
</property>
<property>
<name>urlnormalizer.protocols.file</name>
<value>protocols.txt</value>
- <description>urlnormalizer-protocol configuration file</description>
+ <description>
+ urlnormalizer-protocol configuration file
+ </description>
</property>
<property>
<name>urlnormalizer.protocols.rules</name>
<value></value>
- <description>urlnormalizer-protocol rule definitions: if not empty,
- takes precedence over rules defined in the rule file (see
- urlnormalizer.protocols.file)</description>
+ <description>
+ urlnormalizer-protocol rule definitions: if not empty,
+ takes precedence over rules defined in the rule file (see
+ urlnormalizer.protocols.file)
+ </description>
</property>
@@ -1604,9 +1784,10 @@
<property>
<name>mime.types.file</name>
<value>tika-mimetypes.xml</value>
- <description>Name of file in CLASSPATH containing filename extension and
- magic sequence to mime types mapping information. Overrides the default Tika
config
- if specified.
+ <description>
+ Name of file in CLASSPATH containing filename extension and
+ magic sequence to mime types mapping information. Overrides the default
Tika config
+ if specified.
</description>
</property>
-->
@@ -1614,7 +1795,8 @@
<property>
<name>mime.type.magic</name>
<value>true</value>
- <description>Defines if the mime content type detector uses magic resolution.
+ <description>
+ Defines if the mime content type detector uses magic resolution.
</description>
</property>
@@ -1623,40 +1805,45 @@
<property>
<name>plugin.folders</name>
<value>plugins</value>
- <description>Directories where Nutch plugins are located. Each
- element may be a relative or absolute path. If absolute, it is used
- as is. If relative, it is searched for on the classpath.
- For secure deployments, treat these directories as trusted code: use
- read-only filesystem permissions or immutable images so untrusted
- parties cannot add or replace plugin JARs or plugin.xml files.</description>
+ <description>
+ Directories where Nutch plugins are located. Each
+ element may be a relative or absolute path. If absolute, it is used
+ as is. If relative, it is searched for on the classpath.
+ For secure deployments, treat these directories as trusted code: use
+ read-only filesystem permissions or immutable images so untrusted
+ parties cannot add or replace plugin JARs or plugin.xml files.
+ </description>
</property>
<property>
<name>plugin.auto-activation</name>
<value>true</value>
- <description>Defines if some plugins that are not activated regarding
- the plugin.includes and plugin.excludes properties must be automatically
- activated if they are needed by some active plugins.
+ <description>
+ Defines if some plugins that are not activated regarding
+ the plugin.includes and plugin.excludes properties must be automatically
+ activated if they are needed by some active plugins.
</description>
</property>
<property>
<name>plugin.includes</name>
<value>protocol-http|urlfilter-regex|parse-(html|tika)|index-(basic|anchor)|indexer-solr|scoring-opic|urlnormalizer-(pass|regex|basic)</value>
- <description>Regular expression naming plugin directory names to
- include. Any plugin not matching this expression is excluded.
- By default Nutch includes plugins to crawl HTML and various other
- document formats via HTTP/HTTPS and indexing the crawled content
- into Solr. More plugins are available to support more indexing
- backends, to fetch ftp:// and file:// URLs, for focused crawling,
- and many other use cases.
+ <description>
+ Regular expression naming plugin directory names to
+ include. Any plugin not matching this expression is excluded.
+ By default Nutch includes plugins to crawl HTML and various other
+ document formats via HTTP/HTTPS and indexing the crawled content
+ into Solr. More plugins are available to support more indexing
+ backends, to fetch ftp:// and file:// URLs, for focused crawling,
+ and many other use cases.
</description>
</property>
<property>
<name>plugin.excludes</name>
<value></value>
- <description>Regular expression naming plugin directory names to exclude.
+ <description>
+ Regular expression naming plugin directory names to exclude.
</description>
</property>
@@ -1678,157 +1865,185 @@
<property>
<name>parse.plugin.file</name>
<value>parse-plugins.xml</value>
- <description>The name of the file that defines the associations between
- content-types and parsers.</description>
+ <description>
+ The name of the file that defines the associations between
+ content-types and parsers.
+ </description>
</property>
<property>
<name>parser.character.encoding.default</name>
<value>windows-1252</value>
- <description>The character encoding to fall back to when no other information
- is available</description>
+ <description>
+ The character encoding to fall back to when no other information
+ is available
+ </description>
</property>
<property>
<name>encodingdetector.charset.min.confidence</name>
<value>-1</value>
- <description>A integer between 0-100 indicating minimum confidence value
- for charset auto-detection. Any negative value disables auto-detection.
+ <description>
+ A integer between 0-100 indicating minimum confidence value
+ for charset auto-detection. Any negative value disables auto-detection.
</description>
</property>
<property>
<name>parser.caching.forbidden.policy</name>
<value>content</value>
- <description>If a site (or a page) requests through its robot metatags
- that it should not be shown as cached content, apply this policy. Currently
- three keywords are recognized: "none" ignores any "noarchive" directives.
- "content" doesn't show the content, but shows summaries (snippets).
- "all" doesn't show either content or summaries.</description>
+ <description>
+ If a site (or a page) requests through its robot metatags
+ that it should not be shown as cached content, apply this policy. Currently
+ three keywords are recognized: "none" ignores any "noarchive" directives.
+ "content" doesn't show the content, but shows summaries (snippets).
+ "all" doesn't show either content or summaries.
+ </description>
</property>
<property>
<name>parser.html.impl</name>
<value>neko</value>
- <description>HTML Parser implementation. Currently the following keywords
- are recognized: "neko" uses NekoHTML, "tagsoup" uses TagSoup.
+ <description>
+ HTML Parser implementation. Currently the following keywords
+ are recognized: "neko" uses NekoHTML, "tagsoup" uses TagSoup.
</description>
</property>
<property>
<name>parser.html.form.use_action</name>
<value>false</value>
- <description>If true, HTML parser will collect URLs from form action
- attributes. This may lead to undesirable behavior (submitting empty
- forms during next fetch cycle). If false, form action attribute will
- be ignored.</description>
+ <description>
+ If true, HTML parser will collect URLs from form action
+ attributes. This may lead to undesirable behavior (submitting empty
+ forms during next fetch cycle). If false, form action attribute will
+ be ignored.
+ </description>
</property>
<property>
<name>parser.html.outlinks.ignore_tags</name>
<value></value>
- <description>Comma separated list of HTML tags, from which outlinks
- shouldn't be extracted. Nutch takes links from: a, area, form, frame,
- iframe, script, link, img. If you add any of those tags here, it
- won't be taken. Default is empty list. Probably reasonable value
- for most people would be "img,script,link".</description>
+ <description>
+ Comma separated list of HTML tags, from which outlinks
+ shouldn't be extracted. Nutch takes links from: a, area, form, frame,
+ iframe, script, link, img. If you add any of those tags here, it
+ won't be taken. Default is empty list. Probably reasonable value
+ for most people would be "img,script,link".
+ </description>
</property>
<property>
- <name>parser.html.outlinks.htmlnode_metadata_name</name>
- <value></value>
- <description>if not empty, the source nodename of a found outlink will
- be set in the metadata with this name into the
outlink</description>
+ <name>parser.html.outlinks.htmlnode_metadata_name</name>
+ <value></value>
+ <description>
+ if not empty, the source nodename of a found outlink will
+ be set in the metadata with this name into the outlink
+ </description>
</property>
<property>
<name>parser.html.line.separators</name>
<value>article,aside,blockquote,canvas,dd,div,dl,dt,fieldset,figcaption,figure,footer,form,h1,h2,h3,h4,h5,h6,header,hr,li,main,nav,noscript,ol,output,p,pre,section,table,tfoot,ul,video</value>
- <description>Comma separated list of HTML tags. Newline will be added to the
- parsed text after these tages.
- The default list above are the block-level HTML elements.
- Tags must be in lower case.
- To disable this feature, leave the list empty.</description>
+ <description>
+ Comma separated list of HTML tags. Newline will be added to the
+ parsed text after these tages.
+ The default list above are the block-level HTML elements.
+ Tags must be in lower case.
+ To disable this feature, leave the list empty.
+ </description>
</property>
<property>
<name>htmlparsefilter.order</name>
<value></value>
- <description>The order by which HTMLParse filters are applied.
- If empty, all available HTMLParse filters (as dictated by properties
- plugin-includes and plugin-excludes above) are loaded and applied in system
- defined order. If not empty, only named filters are loaded and applied
- in given order.
- HTMLParse filter ordering MAY have an impact
- on end result, as some filters could rely on the metadata generated by a
previous filter.
+ <description>
+ The order by which HTMLParse filters are applied.
+ If empty, all available HTMLParse filters (as dictated by properties
+ plugin-includes and plugin-excludes above) are loaded and applied in system
+ defined order. If not empty, only named filters are loaded and applied
+ in given order.
+ HTMLParse filter ordering MAY have an impact
+ on end result, as some filters could rely on the metadata generated by a
previous filter.
</description>
</property>
<property>
<name>parsefilter.naivebayes.trainfile</name>
<value>naivebayes-train.txt</value>
- <description>Set the name of the file to be used for Naive Bayes training.
The format will be:
-Each line contains two tab separated parts
-There are two columns/parts:
-1. "1" or "0", "1" for relevant and "0" for irrelevant documents.
-2. Text (text that will be used for training)
+ <description>
+ Set the name of the file to be used for Naive Bayes training. The format
will be:
+ Each line contains two tab separated parts
+ There are two columns/parts:
+ 1. "1" or "0", "1" for relevant and "0" for irrelevant documents.
+ 2. Text (text that will be used for training)
-Each row will be considered a new "document" for the classifier.
-CAUTION: Set the parser.timeout to -1 or a bigger value than 30, when using
this classifier.
+ Each row will be considered a new "document" for the classifier.
+ CAUTION: Set the parser.timeout to -1 or a bigger value than 30, when
using this classifier.
</description>
</property>
<property>
<name>parsefilter.naivebayes.wordlist</name>
<value>naivebayes-wordlist.txt</value>
- <description>Put the name of the file you want to be used as a list of
- important words to be matched in the URL for the model filter. The format
should be one word per line.
+ <description>
+ Put the name of the file you want to be used as a list of
+ important words to be matched in the URL for the model filter. The format
should be one word per line.
</description>
</property>
<property>
<name>parser.timeout</name>
<value>30</value>
- <description>Timeout in seconds for the parsing of a document, otherwise
treats it as an exception and
- moves on the the following documents. This parameter is applied to any
Parser implementation.
- Set to -1 to deactivate, bearing in mind that this could cause
- the parsing to crash because of a very long or corrupted document.
+ <description>
+ Timeout in seconds for the parsing of a document, otherwise treats it as
an exception and
+ moves on the the following documents. This parameter is applied to any
Parser implementation.
+ Set to -1 to deactivate, bearing in mind that this could cause
+ the parsing to crash because of a very long or corrupted document.
</description>
</property>
<property>
<name>parse.filter.urls</name>
<value>true</value>
- <description>Whether the parser will filter URLs (with the configured URL
filters).</description>
+ <description>
+ Whether the parser will filter URLs (with the configured URL filters).
+ </description>
</property>
<property>
<name>parse.normalize.urls</name>
<value>true</value>
- <description>Whether the parser will normalize URLs (with the configured URL
normalizers).</description>
+ <description>
+ Whether the parser will normalize URLs (with the configured URL
normalizers).
+ </description>
</property>
<property>
<name>parser.skip.truncated</name>
<value>true</value>
- <description>Boolean value for whether we should skip parsing for truncated
documents. By default this
- property is activated due to extremely high levels of CPU which parsing can
sometimes take.
+ <description>
+ Boolean value for whether we should skip parsing for truncated documents.
By default this
+ property is activated due to extremely high levels of CPU which parsing
can sometimes take.
</description>
</property>
<property>
<name>parser.delete.failed.parse</name>
<value>false</value>
- <description>Boolean value for whether we should delete a page from the
index when parsing the page fails.
- By default this property is deactivated, because it will delete an existing
page from the index, where a
- previous fetch produced content that was successfully parsed.
+ <description>
+ Boolean value for whether we should delete a page from the index when
parsing the page fails.
+ By default this property is deactivated, because it will delete an
existing page from the index, where a
+ previous fetch produced content that was successfully parsed.
</description>
</property>
<property>
<name>parser.store.text</name>
<value>true</value>
- <description>If true (default value), parser will store parse text
(parse_text directory within the segment).</description>
+ <description>
+ If true (default value), parser will store parse text (parse_text
directory within the segment).
+ </description>
</property>
@@ -1836,23 +2051,27 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<property>
<name>tika.htmlmapper.classname</name>
<value>org.apache.tika.parser.html.IdentityHtmlMapper</value>
- <description>Classname of Tika HTMLMapper to use. Influences the elements
included in the DOM and hence
- the behavior of the HTMLParseFilters.
+ <description>
+ Classname of Tika HTMLMapper to use. Influences the elements included in
the DOM and hence
+ the behavior of the HTMLParseFilters.
</description>
</property>
-->
<property>
- <name>tika.config.file</name>
- <value>tika-config.xml</value>
- <description>Nutch-specific Tika config file</description>
+ <name>tika.config.file</name>
+ <value>tika-config.xml</value>
+ <description>
+ Nutch-specific Tika config file
+ </description>
</property>
<property>
<name>tika.uppercase.element.names</name>
<value>true</value>
- <description>Determines whether TikaParser should uppercase the element name
while generating the DOM
- for a page, as done by Neko (used per default by parse-html)(see NUTCH-1592).
+ <description>
+ Determines whether TikaParser should uppercase the element name while
generating the DOM
+ for a page, as done by Neko (used per default by parse-html)(see
NUTCH-1592).
</description>
</property>
@@ -1860,7 +2079,7 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<name>tika.extractor</name>
<value>none</value>
<description>
- Which text extraction algorithm to use. Valid values are: boilerpipe or none.
+ Which text extraction algorithm to use. Valid values are: boilerpipe or
none.
</description>
</property>
@@ -1868,8 +2087,8 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<name>tika.extractor.boilerpipe.algorithm</name>
<value>ArticleExtractor</value>
<description>
- Which Boilerpipe algorithm to use. Valid values are: DefaultExtractor,
ArticleExtractor
- or CanolaExtractor.
+ Which Boilerpipe algorithm to use. Valid values are: DefaultExtractor,
ArticleExtractor
+ or CanolaExtractor.
</description>
</property>
@@ -1895,88 +2114,104 @@ CAUTION: Set the parser.timeout to -1 or a bigger
value than 30, when using this
<property>
<name>urlfilter.domain.file</name>
<value>domain-urlfilter.txt</value>
- <description>Name of file on CLASSPATH containing either top level domains or
- hostnames used by urlfilter-domain (DomainURLFilter) plugin.</description>
+ <description>
+ Name of file on CLASSPATH containing either top level domains or
+ hostnames used by urlfilter-domain (DomainURLFilter) plugin.
+ </description>
</property>
<property>
<name>urlfilter.regex.file</name>
<value>regex-urlfilter.txt</value>
- <description>Name of file on CLASSPATH containing regular expressions
- used by urlfilter-regex (RegexURLFilter) plugin.</description>
+ <description>
+ Name of file on CLASSPATH containing regular expressions
+ used by urlfilter-regex (RegexURLFilter) plugin.
+ </description>
</property>
<property>
<name>urlfilter.automaton.file</name>
<value>automaton-urlfilter.txt</value>
- <description>Name of file on CLASSPATH containing regular expressions
- used by urlfilter-automaton (AutomatonURLFilter) plugin.</description>
+ <description>
+ Name of file on CLASSPATH containing regular expressions
+ used by urlfilter-automaton (AutomatonURLFilter) plugin.
+ </description>
</property>
<property>
<name>urlfilter.prefix.file</name>
<value>prefix-urlfilter.txt</value>
- <description>Name of file on CLASSPATH containing URL prefixes
- used by urlfilter-prefix (PrefixURLFilter) plugin.</description>
+ <description>
+ Name of file on CLASSPATH containing URL prefixes
+ used by urlfilter-prefix (PrefixURLFilter) plugin.
+ </description>
</property>
<property>
<name>urlfilter.suffix.file</name>
<value>suffix-urlfilter.txt</value>
- <description>Name of file on CLASSPATH containing URL suffixes
- used by urlfilter-suffix (SuffixURLFilter) plugin.</description>
+ <description>
+ Name of file on CLASSPATH containing URL suffixes
+ used by urlfilter-suffix (SuffixURLFilter) plugin.
+ </description>
</property>
<property>
<name>urlfilter.fast.file</name>
<value>fast-urlfilter.txt</value>
- <description>Name of file containing rules and regular expressions
- used by urlfilter-fast (FastURLFilter) plugin. If the filename
- includes a scheme (for example, hdfs://) it is loaded using the
- Hadoop FileSystem implementation supporting that scheme. If the
- filename does not contain a scheme, the file is loaded from
- CLASSPATH. If indicated by file extension (.gz, .bzip2, .zst),
- the file is decompressed while reading using Hadoop-provided
- compression codecs.</description>
+ <description>
+ Name of file containing rules and regular expressions
+ used by urlfilter-fast (FastURLFilter) plugin. If the filename
+ includes a scheme (for example, hdfs://) it is loaded using the
+ Hadoop FileSystem implementation supporting that scheme. If the
+ filename does not contain a scheme, the file is loaded from
+ CLASSPATH. If indicated by file extension (.gz, .bzip2, .zst),
+ the file is decompressed while reading using Hadoop-provided
+ compression codecs.
+ </description>
</property>
<property>
<name>urlfilter.fast.url.max.length</name>
<value>-1</value>
- <description>Filters URLs based on their overall length.
- The default value of -1 means that it is deactivated.
+ <description>
+ Filters URLs based on their overall length.
+ The default value of -1 means that it is deactivated.
</description>
</property>
<property>
<name>urlfilter.fast.url.path.max.length</name>
<value>-1</value>
- <description>Filters URLs based on the length of their path element.
- The default value of -1 means that it is deactivated.
+ <description>
+ Filters URLs based on the length of their path element.
+ The default value of -1 means that it is deactivated.
</description>
</property>
<property>
<name>urlfilter.fast.url.query.max.length</name>
<value>-1</value>
- <description>Filters URLs based on the length of their query element.
- The default value of -1 means that it is deactivated.
+ <description>
+ Filters URLs based on the length of their query element.
+ The default value of -1 means that it is deactivated.
</description>
</property>
<property>
<name>urlfilter.order</name>
<value></value>
- <description>The order by which URL filters are applied.
- If empty, all available URL filters (as dictated by properties
- plugin-includes and plugin-excludes above) are loaded and applied in system
- defined order. If not empty, only named filters are loaded and applied
- in given order. For example, if this property has value:
- org.apache.nutch.urlfilter.regex.RegexURLFilter
org.apache.nutch.urlfilter.prefix.PrefixURLFilter
- then RegexURLFilter is applied first, and PrefixURLFilter second.
- Since all filters are AND'ed, filter ordering does not have impact
- on end result, but it may have performance implication, depending
- on relative expensiveness of filters.
+ <description>
+ The order by which URL filters are applied.
+ If empty, all available URL filters (as dictated by properties
+ plugin-includes and plugin-excludes above) are loaded and applied in system
+ defined order. If not empty, only named filters are loaded and applied
+ in given order. For example, if this property has value:
+ org.apache.nutch.urlfilter.regex.RegexURLFilter
org.apache.nutch.urlfilter.prefix.PrefixURLFilter
+ then RegexURLFilter is applied first, and PrefixURLFilter second.
+ Since all filters are AND'ed, filter ordering does not have impact
+ on end result, but it may have performance implication, depending
+ on relative expensiveness of filters.
</description>
</property>
@@ -1985,10 +2220,11 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<property>
<name>scoring.filter.order</name>
<value></value>
- <description>The order in which scoring filters are applied. This
- may be left empty (in which case all available scoring filters will
- be applied in system defined order), or a space separated list of
- implementation classes.
+ <description>
+ The order in which scoring filters are applied. This
+ may be left empty (in which case all available scoring filters will
+ be applied in system defined order), or a space separated list of
+ implementation classes.
</description>
</property>
@@ -2000,28 +2236,31 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<property>
<name>scoring.depth.max</name>
<value>1000</value>
- <description>Max depth value from seed allowed by default.
- Can be overridden on a per-seed basis by specifying "_maxdepth_=VALUE"
- as a seed metadata. This plugin adds a "_depth_" metadatum to the pages
- to track the distance from the seed it was found from.
- The depth is used to prioritise URLs in the generation step so that
- shallower pages are fetched first.
+ <description>
+ Max depth value from seed allowed by default.
+ Can be overridden on a per-seed basis by specifying "_maxdepth_=VALUE"
+ as a seed metadata. This plugin adds a "_depth_" metadatum to the pages
+ to track the distance from the seed it was found from.
+ The depth is used to prioritise URLs in the generation step so that
+ shallower pages are fetched first.
</description>
</property>
<property>
<name>scoring.depth.override.pattern</name>
<value></value>
- <description>URLs matching this pattern pass a different max depth value
- to their outlinks configured in scoring.depth.max.override.
+ <description>
+ URLs matching this pattern pass a different max depth value
+ to their outlinks configured in scoring.depth.max.override.
</description>
</property>
<property>
<name>scoring.depth.max.override</name>
<value></value>
- <description>This max depth value is passed to outlinks matching the pattern
- configured in scoring.depth.override.pattern.
+ <description>
+ This max depth value is passed to outlinks matching the pattern
+ configured in scoring.depth.override.pattern.
</description>
</property>
@@ -2033,36 +2272,40 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
-->
<property>
- <name>scoring.similarity.model</name>
- <value>cosine</value>
- <description>The type of similarity metric to use. Eg - cosine (which is,
currently, the only available model).
- Please make sure to set the model specific properties for the scoring to
function properly.
- Description of these properties can be found on the wiki.
- </description>
+ <name>scoring.similarity.model</name>
+ <value>cosine</value>
+ <description>
+ The type of similarity metric to use. Eg - cosine (which is, currently,
the only available model).
+ Please make sure to set the model specific properties for the scoring to
function properly.
+ Description of these properties can be found on the wiki.
+ </description>
</property>
- <property>
+<property>
<name>scoring.similarity.ngrams</name>
<value>1,1</value>
- <description>Specifies the min 'n' and max 'n' in ngrams as comma-separated.
+ <description>
+ Specifies the min 'n' and max 'n' in ngrams as comma-separated.
If one value is specified as 'n', it will be used for both the min 'n' and
max 'n' in ngrams.
</description>
</property>
<property>
- <name>cosine.goldstandard.file</name>
- <value>goldstandard.txt</value>
- <description>Path to the gold standard file which contains all the
relevant text and terms,
- pertaining to the domain.
- </description>
+ <name>cosine.goldstandard.file</name>
+ <value>goldstandard.txt</value>
+ <description>
+ Path to the gold standard file which contains all the relevant text and
terms,
+ pertaining to the domain.
+ </description>
</property>
- <property>
- <name>scoring.similarity.stopword.file</name>
- <value>stopwords.txt</value>
- <description>Name of the stopword text file. The user can specify a custom
list of stop words
- in a text file. Each new stopword should be on a new line.
- </description>
+<property>
+ <name>scoring.similarity.stopword.file</name>
+ <value>stopwords.txt</value>
+ <description>
+ Name of the stopword text file. The user can specify a custom list of stop
words
+ in a text file. Each new stopword should be on a new line.
+ </description>
</property>
<!-- scoring filter orphan properties -->
@@ -2070,16 +2313,18 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<property>
<name>scoring.orphan.mark.gone.after</name>
<value>2592000</value>
- <description>Time in seconds after which orphaned
- pages are marked as gone. Default is 30 days.
+ <description>
+ Time in seconds after which orphaned
+ pages are marked as gone. Default is 30 days.
</description>
</property>
<property>
<name>scoring.orphan.mark.orphan.after</name>
<value>3456000</value>
- <description>Time in seconds after which orphaned
- pages are marked as gone. Default is 40 days.
+ <description>
+ Time in seconds after which orphaned
+ pages are marked as gone. Default is 40 days.
</description>
</property>
@@ -2092,7 +2337,7 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<name>scoring.db.md</name>
<value></value>
<description>
- Comma-separated list of keys to be taken from CrawlDb metadata of a URL to
the fetched content metadata.
+ Comma-separated list of keys to be taken from CrawlDb metadata of a URL to
the fetched content metadata.
</description>
</property>
@@ -2100,7 +2345,7 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<name>scoring.content.md</name>
<value></value>
<description>
- Comma-separated list of keys to be taken from content metadata of a URL and
put as metadata in the parse data.
+ Comma-separated list of keys to be taken from content metadata of a URL
and put as metadata in the parse data.
</description>
</property>
@@ -2108,7 +2353,7 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<name>scoring.parse.md</name>
<value></value>
<description>
- Comma-separated list of keys to be taken from metadata of the parse data of
a URL and propagated as metadata to the URL outlinks.
+ Comma-separated list of keys to be taken from metadata of the parse data
of a URL and propagated as metadata to the URL outlinks.
</description>
</property>
@@ -2119,46 +2364,50 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<property>
<name>lang.analyze.max.length</name>
<value>2048</value>
- <description> The maximum number of bytes used to identify
- the language (0 means full content analysis).
- The larger is this value, the better is the analysis, but the
- slowest it is.
+ <description>
+ The maximum number of bytes used to identify
+ the language (0 means full content analysis).
+ The larger is this value, the better is the analysis, but the
+ slowest it is.
</description>
</property>
<property>
<name>lang.extraction.policy</name>
<value>detect,identify</value>
- <description>This determines when the plugin uses detection and
- statistical identification mechanisms. The order in which the
- detect and identify are written will determine the extraction
- policy. Default case (detect,identify) means the plugin will
- first try to extract language info from page headers and metadata,
- if this is not successful it will try using tika language
- identification. Possible values are:
- detect
- identify
- detect,identify
- identify,detect
+ <description>
+ This determines when the plugin uses detection and
+ statistical identification mechanisms. The order in which the
+ detect and identify are written will determine the extraction
+ policy. Default case (detect,identify) means the plugin will
+ first try to extract language info from page headers and metadata,
+ if this is not successful it will try using tika language
+ identification. Possible values are:
+ detect
+ identify
+ detect,identify
+ identify,detect
</description>
</property>
<property>
<name>lang.identification.only.certain</name>
<value>false</value>
- <description>If set to true with lang.extraction.policy containing identify,
- the language code returned by Tika will be assigned to the document ONLY
- if it is deemed certain by Tika.
+ <description>
+ If set to true with lang.extraction.policy containing identify,
+ the language code returned by Tika will be assigned to the document ONLY
+ if it is deemed certain by Tika.
</description>
</property>
<property>
<name>lang.index.languages</name>
<value></value>
- <description>If not empty, should be a comma separated list of language
codes.
- Only documents with one of these language codes will be indexed.
- "unknown" is a valid language code, will match documents where language
- detection failed.
+ <description>
+ If not empty, should be a comma separated list of language codes.
+ Only documents with one of these language codes will be indexed.
+ "unknown" is a valid language code, will match documents where language
+ detection failed.
</description>
</property>
@@ -2167,29 +2416,31 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<property>
<name>index.jexl.filter</name>
<value></value>
- <description> A JEXL expression. If it evaluates to false,
- the document will not be indexed.
- Available primitives in the JEXL context:
- * status, fetchTime, modifiedTime, retries, interval, score, signature, url,
text, title
- Available objects in the JEXL context:
- * httpStatus - contains majorCode, minorCode, message
- * documentMeta, contentMeta, parseMeta - contain all the Metadata properties.
+ <description>
+ A JEXL expression. If it evaluates to false,
+ the document will not be indexed.
+ Available primitives in the JEXL context:
+ * status, fetchTime, modifiedTime, retries, interval, score, signature,
url, text, title
+ Available objects in the JEXL context:
+ * httpStatus - contains majorCode, minorCode, message
+ * documentMeta, contentMeta, parseMeta - contain all the Metadata
properties.
each property value is always an array of Strings (so if you expect one
value, use [0])
- * doc - contains all the NutchFields from the NutchDocument.
+ * doc - contains all the NutchFields from the NutchDocument.
each property value is always an array of Objects.
- Expressions are evaluated in a sandboxed JEXL engine (see also
- nutch.jexl.disable.sandbox).
+ Expressions are evaluated in a sandboxed JEXL engine (see also
+ nutch.jexl.disable.sandbox).
</description>
</property>
<property>
<name>nutch.jexl.disable.sandbox</name>
<value>false</value>
- <description>If true, disables the Commons JEXL sandbox and the restriction
- on the JEXL "new" operator for all Nutch JEXL expressions (index filter,
- generator, hostdb filter, crawl_db_reader, exchange-jexl, etc.). This is
- unsafe and should only be used in fully trusted environments when a
- legitimate expression cannot be expressed under the default sandbox.
+ <description>
+ If true, disables the Commons JEXL sandbox and the restriction
+ on the JEXL "new" operator for all Nutch JEXL expressions (index filter,
+ generator, hostdb filter, crawl_db_reader, exchange-jexl, etc.). This is
+ unsafe and should only be used in fully trusted environments when a
+ legitimate expression cannot be expressed under the default sandbox.
</description>
</property>
@@ -2199,12 +2450,12 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<name>index.static</name>
<value></value>
<description>
- Used by plugin index-static to adds fields with static data at indexing time.
- You can specify a comma-separated list of fieldname:fieldcontent per Nutch
job.
- Each fieldcontent can have multiple values separated by space, e.g.,
- field1:value1.1 value1.2 value1.3,field2:value2.1 value2.2 ...
- It can be useful when collections can't be created by URL patterns,
- like in subcollection, but on a job-basis.
+ Used by plugin index-static to adds fields with static data at indexing
time.
+ You can specify a comma-separated list of fieldname:fieldcontent per Nutch
job.
+ Each fieldcontent can have multiple values separated by space, e.g.,
+ field1:value1.1 value1.2 value1.3,field2:value2.1 value2.2 ...
+ It can be useful when collections can't be created by URL patterns,
+ like in subcollection, but on a job-basis.
</description>
</property>
@@ -2212,8 +2463,8 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<name>index.static.fieldsep</name>
<value>,</value>
<description>
- Used by plugin index-static to parse the property index.static. Default:
comma.
- This delimiter is used to separate individual field specifications in the
property.
+ Used by plugin index-static to parse the property index.static. Default:
comma.
+ This delimiter is used to separate individual field specifications in the
property.
</description>
</property>
@@ -2221,8 +2472,8 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<name>index.static.keysep</name>
<value>:</value>
<description>
- Used by plugin index-static to parse the property index.static. Default:
colon.
- This delimiter is used to separate the field name from the field value in
the field specification.
+ Used by plugin index-static to parse the property index.static. Default:
colon.
+ This delimiter is used to separate the field name from the field value in
the field specification.
</description>
</property>
@@ -2230,8 +2481,8 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<name>index.static.valuesep</name>
<value> </value>
<description>
- Used by plugin index-static to parse the property index.static. Default:
space.
- This delimiter is used to separate multiple field values in the value
setting of the field specification.
+ Used by plugin index-static to parse the property index.static. Default:
space.
+ This delimiter is used to separate multiple field values in the value
setting of the field specification.
</description>
</property>
@@ -2242,9 +2493,9 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<name>index.parse.md</name>
<value>metatag.description,metatag.keywords</value>
<description>
- Comma-separated list of keys to be taken from the parse metadata to generate
fields.
- Can be used e.g. for 'description' or 'keywords' provided that these values
are generated
- by a parser (see parse-metatags plugin)
+ Comma-separated list of keys to be taken from the parse metadata to
generate fields.
+ Can be used e.g. for 'description' or 'keywords' provided that these
values are generated
+ by a parser (see parse-metatags plugin)
</description>
</property>
@@ -2290,8 +2541,8 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<name>index.geoip.db.anonymous</name>
<value></value>
<description>
- GeoIP2 Anonymous IP database file (MMDB format).
- Identifies anonymous proxies, VPNs, Tor exit nodes, and hosting providers.
+ GeoIP2 Anonymous IP database file (MMDB format).
+ Identifies anonymous proxies, VPNs, Tor exit nodes, and hosting providers.
</description>
</property>
@@ -2299,8 +2550,8 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<name>index.geoip.db.asn</name>
<value></value>
<description>
- GeoIP2/GeoLite2 ASN database file (MMDB format).
- Provides autonomous system number and organization information.
+ GeoIP2/GeoLite2 ASN database file (MMDB format).
+ Provides autonomous system number and organization information.
</description>
</property>
@@ -2308,8 +2559,8 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<name>index.geoip.db.city</name>
<value></value>
<description>
- GeoIP2/GeoLite2 City database file (MMDB format).
- Provides city, subdivision, country, continent, and location data.
+ GeoIP2/GeoLite2 City database file (MMDB format).
+ Provides city, subdivision, country, continent, and location data.
</description>
</property>
@@ -2317,8 +2568,8 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<name>index.geoip.db.connection</name>
<value></value>
<description>
- GeoIP2 Connection Type database file (MMDB format).
- Identifies connection type: Cable/DSL, Cellular, Corporate, or Satellite.
+ GeoIP2 Connection Type database file (MMDB format).
+ Identifies connection type: Cable/DSL, Cellular, Corporate, or Satellite.
</description>
</property>
@@ -2326,10 +2577,10 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<name>index.geoip.db.country</name>
<value></value>
<description>
- GeoIP2/GeoLite2 Country database file (MMDB format).
- Provides country, continent, and represented country information.
- This is a lighter-weight alternative to the City database when only
- country-level information is needed.
+ GeoIP2/GeoLite2 Country database file (MMDB format).
+ Provides country, continent, and represented country information.
+ This is a lighter-weight alternative to the City database when only
+ country-level information is needed.
</description>
</property>
@@ -2337,8 +2588,8 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<name>index.geoip.db.domain</name>
<value></value>
<description>
- GeoIP2 Domain database file (MMDB format).
- Provides the second-level domain associated with the IP address.
+ GeoIP2 Domain database file (MMDB format).
+ Provides the second-level domain associated with the IP address.
</description>
</property>
@@ -2346,8 +2597,8 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<name>index.geoip.db.isp</name>
<value></value>
<description>
- GeoIP2 ISP database file (MMDB format).
- Provides ISP, organization, and autonomous system information.
+ GeoIP2 ISP database file (MMDB format).
+ Provides ISP, organization, and autonomous system information.
</description>
</property>
@@ -2356,9 +2607,9 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<name>index.geoip.insights.userid</name>
<value></value>
<description>
- The userId for the MaxMind GeoIP2 Precision Insights web service.
- If set along with licensekey, the Insights service will be queried in
addition
- to any configured local databases.
+ The userId for the MaxMind GeoIP2 Precision Insights web service.
+ If set along with licensekey, the Insights service will be queried in
addition
+ to any configured local databases.
</description>
</property>
@@ -2366,14 +2617,15 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<name>index.geoip.insights.licensekey</name>
<value></value>
<description>
- The license key for the MaxMind GeoIP2 Precision Insights web service.
+ The license key for the MaxMind GeoIP2 Precision Insights web service.
</description>
</property>
<property>
<name>index.replace.regexp</name>
- <value/>
- <description>Allows indexing-time regexp replace manipulation of metadata
fields.
+ <value></value>
+ <description>
+ Allows indexing-time regexp replace manipulation of metadata fields.
The format of the property is a list of regexp replacements, one line per
field being
modified. Include index-replace in your plugin.includes.
@@ -2392,7 +2644,8 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<property>
<name>index.arbitrary.function.count</name>
<value></value>
- <description>The count of arbitrary additions/edits to the document.
+ <description>
+ The count of arbitrary additions/edits to the document.
Specify the remaining properties (fieldName, className, constructorArgs,
methodName, and methodArgs) independently in this file by appending a
dot (.) followed by integer numerals (beginning with '0') to the property
@@ -2409,21 +2662,26 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<property>
<name>index.arbitrary.fieldName.0</name>
<value></value>
- <description>The name of the field to add to the document with the value
- returned from the custom POJO.</description>
+ <description>
+ The name of the field to add to the document with the value
+ returned from the custom POJO.
+ </description>
</property>
<property>
<name>index.arbitrary.className.0</name>
<value></value>
- <description>The fully qualified name of the POJO class that will supply
- values for the new field.</description>
+ <description>
+ The fully qualified name of the POJO class that will supply
+ values for the new field.
+ </description>
</property>
<property>
<name>index.arbitrary.constructorArgs.0</name>
<value></value>
- <description>The values (as strings) to pass into the POJO constructor.
+ <description>
+ The values (as strings) to pass into the POJO constructor.
The POJO must accept a String representation of the NutchDocument's URL
as the first parameter in the constructor. The values you specify here
will populate the constructor arguments 1,..,n-1 where n=the count of
@@ -2434,45 +2692,54 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<property>
<name>index.arbitrary.methodName.0</name>
<value></value>
- <description>The name of the method to invoke on the instance of your custom
- class in order to determine the value to add to the document.</description>
- </property>
+ <description>
+ The name of the method to invoke on the instance of your custom
+ class in order to determine the value to add to the document.
+ </description>
+</property>
<property>
<name>index.arbitrary.methodArgs.0</name>
<value></value>
- <description>The values (as strings) to pass into the named method on the
POJO
+ <description>
+ The values (as strings) to pass into the named method on the POJO
instance. Unlike the constructor args, there is no required argument that
this
method in the POJO must accept, i.e., the Arbitrary Indexer doesn't supply
any
- arguments taken from the NutchDocument values by default.</description>
+ arguments taken from the NutchDocument values by default.
+ </description>
</property>
<property>
<name>index.arbitrary.overwrite.0</name>
<value></value>
- <description>Whether to overwrite any existing value in the doc for
- for fieldName. Default is false if not specified in config</description>
+ <description>
+ Whether to overwrite any existing value in the doc for
+ for fieldName. Default is false if not specified in config
+ </description>
</property>
<property>
<name>index.arbitrary.all.fields.access.0</name>
<value></value>
- <description>Whether to pass all indexing filter fields to constructor
- for the user POJO. These are the NutchDocument doc, Parse parse,
- url (as org.apache.hadoop.io.Text), CrawlDatum datum, and
- Inlinks inlinks objects. Default value is false to preserve
- compatibility with previous index-arbitrary behavior.</description>
+ <description>
+ Whether to pass all indexing filter fields to constructor
+ for the user POJO. These are the NutchDocument doc, Parse parse,
+ url (as org.apache.hadoop.io.Text), CrawlDatum datum, and
+ Inlinks inlinks objects. Default value is false to preserve
+ compatibility with previous index-arbitrary behavior.
+ </description>
</property>
<!-- parse-metatags plugin properties -->
<property>
<name>metatags.names</name>
<value>description,keywords</value>
- <description> Names of the metatags to extract, separated by ','.
- Use '*' to extract all metatags. Prefixes the names with 'metatag.'
- in the parse-metadata. For instance to index description and keywords,
- you need to activate the plugin index-metadata and set the value of the
- parameter 'index.parse.md' to 'metatag.description,metatag.keywords'.
+ <description>
+ Names of the metatags to extract, separated by ','.
+ Use '*' to extract all metatags. Prefixes the names with 'metatag.'
+ in the parse-metadata. For instance to index description and keywords,
+ you need to activate the plugin index-metadata and set the value of the
+ parameter 'index.parse.md' to 'metatag.description,metatag.keywords'.
</description>
</property>
@@ -2482,10 +2749,12 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<name>io.serializations</name>
<value>org.apache.hadoop.io.serializer.WritableSerialization,org.apache.hadoop.io.serializer.JavaSerialization</value>
<!-- org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization,
- org.apache.hadoop.io.serializer.avro.AvroReflectSerialization,
- org.apache.hadoop.io.serializer.avro.AvroGenericSerialization, -->
- <description>A list of serialization classes that can be used for
- obtaining serializers and deserializers.</description>
+ org.apache.hadoop.io.serializer.avro.AvroReflectSerialization,
+ org.apache.hadoop.io.serializer.avro.AvroGenericSerialization, -->
+ <description>
+ A list of serialization classes that can be used for
+ obtaining serializers and deserializers.
+ </description>
</property>
<!-- linkrank scoring properties -->
@@ -2493,63 +2762,82 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<property>
<name>link.ignore.internal.host</name>
<value>true</value>
- <description>Ignore outlinks to the same hostname.</description>
+ <description>
+ Ignore outlinks to the same hostname.
+ </description>
</property>
<property>
<name>link.ignore.internal.domain</name>
<value>true</value>
- <description>Ignore outlinks to the same domain.</description>
+ <description>
+ Ignore outlinks to the same domain.
+ </description>
</property>
<property>
<name>link.ignore.limit.page</name>
<value>true</value>
- <description>Limit to only a single outlink to the same page.</description>
+ <description>
+ Limit to only a single outlink to the same page.
+ </description>
</property>
<property>
<name>link.ignore.limit.domain</name>
<value>true</value>
- <description>Limit to only a single outlink to the same domain.</description>
+ <description>
+ Limit to only a single outlink to the same domain.
+ </description>
</property>
<property>
<name>link.analyze.num.iterations</name>
<value>10</value>
- <description>The number of LinkRank iterations to run.</description>
+ <description>
+ The number of LinkRank iterations to run.
+ </description>
</property>
<property>
<name>link.analyze.initial.score</name>
<value>1.0f</value>
- <description>The initial score.</description>
+ <description>
+ The initial score.
+ </description>
</property>
<property>
<name>link.analyze.damping.factor</name>
<value>0.85f</value>
- <description>The damping factor.</description>
+ <description>
+ The damping factor.
+ </description>
</property>
<property>
<name>link.delete.gone</name>
<value>false</value>
- <description>Whether to delete gone pages from the web graph. Gone pages
include redirects and duplicates.</description>
+ <description>
+ Whether to delete gone pages from the web graph. Gone pages include
redirects and duplicates.
+ </description>
</property>
<property>
<name>link.score.updater.clear.score</name>
<value>0.0f</value>
- <description>The default score for URLs that are not in the web
graph.</description>
+ <description>
+ The default score for URLs that are not in the web graph.
+ </description>
</property>
<property>
<name>mapreduce.fileoutputcommitter.marksuccessfuljobs</name>
<value>false</value>
- <description>Hadoop >= 0.21 generates SUCCESS files in the output which can
crash
- the readers. This should not be an issue once Nutch is ported to the new
MapReduce API
- but for now this parameter should prevent such cases.
+ <description>
+ Hadoop >= 0.21 generates SUCCESS files in the output which can crash
+ the readers. This should not be an issue once Nutch is ported to the new
MapReduce API
+ but for now this parameter should prevent such cases.
</description>
</property>
@@ -2559,7 +2847,7 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<name>subcollection.default.fieldname</name>
<value>subcollection</value>
<description>
- The default field name for the subcollections.
+ The default field name for the subcollections.
</description>
</property>
@@ -2567,7 +2855,7 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<name>subcollection.case.insensitive</name>
<value>false</value>
<description>
- Whether the URL prefixes are to be treated case insensitive.
+ Whether the URL prefixes are to be treated case insensitive.
</description>
</property>
@@ -2576,13 +2864,17 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<property>
<name>headings</name>
<value>h1,h2</value>
- <description>Comma separated list of headings to retrieve from the
document</description>
+ <description>
+ Comma separated list of headings to retrieve from the document
+ </description>
</property>
<property>
<name>headings.multivalued</name>
<value>false</value>
- <description>Whether to support multivalued headings.</description>
+ <description>
+ Whether to support multivalued headings.
+ </description>
</property>
<!-- mimetype-filter plugin properties -->
@@ -2680,31 +2972,40 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<property>
<name>selenium.hub.port</name>
<value>4444</value>
- <description>Selenium Hub Location connection port</description>
+ <description>
+ Selenium Hub Location connection port
+ </description>
</property>
<property>
<name>selenium.hub.path</name>
<value>/wd/hub</value>
- <description>Selenium Hub Location connection path</description>
+ <description>
+ Selenium Hub Location connection path
+ </description>
</property>
<property>
<name>selenium.hub.host</name>
<value>localhost</value>
- <description>Selenium Hub Location connection host</description>
+ <description>
+ Selenium Hub Location connection host
+ </description>
</property>
<property>
<name>selenium.hub.protocol</name>
<value>http</value>
- <description>Selenium Hub Location connection protocol</description>
+ <description>
+ Selenium Hub Location connection protocol
+ </description>
</property>
<property>
<name>selenium.grid.driver</name>
<value>firefox</value>
- <description>A String value representing the flavour of Selenium
+ <description>
+ A String value representing the flavour of Selenium
WebDriver() used on the selenium grid. We must set `selenium.driver` to
`remote` first.
Currently the following options
exist - 'firefox', 'chrome', 'random' </description>
@@ -2713,16 +3014,18 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<property>
<name>selenium.grid.binary</name>
<value></value>
- <description>A String value representing the path to the browser binary
+ <description>
+ A String value representing the path to the browser binary
location for each node
- </description>
+ </description>
</property>
<!-- headless options for Firefox and Chrome-->
<property>
<name>selenium.enable.headless</name>
<value>false</value>
- <description>A Boolean value representing the headless option
+ <description>
+ A Boolean value representing the headless option
for Firefix and Chrome drivers
</description>
</property>
@@ -2731,56 +3034,63 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<property>
<name>selenium.firefox.allowed.hosts</name>
<value>localhost</value>
- <description>A String value representing the allowed hosts preference
- according to the operating system hosts file (Example - /etc/hosts in Unix).
- Currently this option exist for - 'firefox' </description>
+ <description>
+ A String value representing the allowed hosts preference
+ according to the operating system hosts file (Example - /etc/hosts in
Unix).
+ Currently this option exist for - 'firefox' </description>
</property>
<property>
<name>selenium.firefox.binary.timeout</name>
<value>45</value>
- <description>A Long value representing the timeout value
- for firefox to be available for command execution. The value is in seconds.
- Currently this option exist for - 'firefox' </description>
+ <description>
+ A Long value representing the timeout value
+ for firefox to be available for command execution. The value is in seconds.
+ Currently this option exist for - 'firefox' </description>
</property>
<property>
<name>selenium.firefox.enable.flash</name>
<value>false</value>
- <description>A Boolean value representing if flash should
- be enabled or disabled. The default value is disabled.
- Currently this option exist for - 'firefox' </description>
+ <description>
+ A Boolean value representing if flash should
+ be enabled or disabled. The default value is disabled.
+ Currently this option exist for - 'firefox' </description>
</property>
<property>
<name>selenium.firefox.load.image</name>
<value>1</value>
- <description>An Integer value representing the restriction on
- loading images. The default value is no restriction i.e. load all images.
- Other options are:
- 1: Load all images, regardless of origin
- 2: Block all images
- 3: Prevent third-party images from loading
- Currently this option exist for - 'firefox' </description>
+ <description>
+ An Integer value representing the restriction on
+ loading images. The default value is no restriction i.e. load all images.
+ Other options are:
+ 1: Load all images, regardless of origin
+ 2: Block all images
+ 3: Prevent third-party images from loading
+ Currently this option exist for - 'firefox' </description>
</property>
<property>
<name>selenium.firefox.load.stylesheet</name>
<value>1</value>
- <description>An Integer value representing the restriction on
- loading stylesheet. The default value is no restriction i.e. load
- all stylesheet.
- Other options are:
- 1: Load all stylesheet
- 2: Block all stylesheet
- Currently this option exist for - 'firefox' </description>
+ <description>
+ An Integer value representing the restriction on
+ loading stylesheet. The default value is no restriction i.e. load
+ all stylesheet.
+ Other options are:
+ 1: Load all stylesheet
+ 2: Block all stylesheet
+ Currently this option exist for - 'firefox' </description>
</property>
<!-- selenium chrome configurations -->
<property>
<name>webdriver.chrome.driver</name>
<value>/root/chromedriver</value>
- <description>The path to the ChromeDriver binary</description>
+ <description>
+ The path to the ChromeDriver binary
+ </description>
</property>
<!-- end of selenium chrome configurations -->
@@ -2800,12 +3110,13 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
<property>
<name>store.ip.address</name>
<value>false</value>
- <description>Enables us to capture the specific IP address
- (InetSocketAddress) of the host which we connect to via the given
- protocol. Currently supported by: protocol-ftp, protocol-http,
- protocol-okhttp, protocol-htmlunit, protocol-selenium. Note that
- the IP address is required by the plugin index-geoip and when
- writing WARC files.
+ <description>
+ Enables us to capture the specific IP address
+ (InetSocketAddress) of the host which we connect to via the given
+ protocol. Currently supported by: protocol-ftp, protocol-http,
+ protocol-okhttp, protocol-htmlunit, protocol-selenium. Note that
+ the IP address is required by the plugin index-geoip and when
+ writing WARC files.
</description>
</property>
@@ -3139,7 +3450,7 @@ one publisher implementation for RabbitMQ (plugin
publish-rabbitmq).
<value>true</value>
<description>
Normalize URLs from sitemaps.
- </description>
+ </description>
</property>
<property>
@@ -3148,7 +3459,7 @@ one publisher implementation for RabbitMQ (plugin
publish-rabbitmq).
<description>
Always try <host>/sitemap.xml root even if no sitemap
is announced in /robots.txt.
- </description>
+ </description>
</property>
<property>
@@ -3160,7 +3471,7 @@ one publisher implementation for RabbitMQ (plugin
publish-rabbitmq).
these values may have unexpected effects on what is crawled. Use this
only if you can trust the sitemap and if the values in the sitemap do
fit with your crawler configuration.
- </description>
+ </description>
</property>
<property>
@@ -3168,7 +3479,7 @@ one publisher implementation for RabbitMQ (plugin
publish-rabbitmq).
<value>3</value>
<description>
Maximum number of redirects to follow.
- </description>
+ </description>
</property>
<property>
@@ -3176,7 +3487,7 @@ one publisher implementation for RabbitMQ (plugin
publish-rabbitmq).
<value>52428800</value>
<description>
Maximum sitemap size in bytes.
- </description>
+ </description>
</property>
</configuration>
