[
https://issues.apache.org/jira/browse/NUTCH-1966?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14368443#comment-14368443
]
Chris A. Mattmann commented on NUTCH-1966:
------------------------------------------
OK tested config endpoints, working fine:
{noformat}
[chipotle:~] mattmann% curl http://localhost:8081/config
["default"][chipotle:~] mattmann% curl http://localhost:8081/config/default
{"anchorIndexingFilter.deduplicate":"false","crawl.gen.delay":"604800000","db.fetch.interval.default":"2592000","db.fetch.interval.max":"7776000","db.fetch.retry.max":"3","db.fetch.schedule.adaptive.dec_rate":"0.2","db.fetch.schedule.adaptive.inc_rate":"0.4","db.fetch.schedule.adaptive.max_interval":"31536000.0","db.fetch.schedule.adaptive.min_interval":"60.0","db.fetch.schedule.adaptive.sync_delta":"true","db.fetch.schedule.adaptive.sync_delta_rate":"0.3","db.fetch.schedule.class":"org.apache.nutch.crawl.DefaultFetchSchedule","db.fetch.schedule.mime.file":"adaptive-mimetypes.txt","db.ignore.external.links":"false","db.ignore.internal.links":"true","db.injector.overwrite":"false","db.injector.update":"false","db.max.anchor.length":"100","db.max.inlinks":"10000","db.max.outlinks.per.page":"100","db.preserve.backup":"true","db.score.count.filtered":"false","db.score.injected":"1.0","db.score.link.external":"1.0","db.score.link.internal":"1.0","db.signature.class":"org.apache.nutch.crawl.MD5Signature","db.signature.text_profile.min_token_len":"2","db.signature.text_profile.quant_rate":"0.01","db.update.additions.allowed":"true","db.update.max.inlinks":"10000","db.update.purge.404":"false","db.url.filters":"false","db.url.normalizers":"false","elastic.index":"nutch","elastic.max.bulk.docs":"250","elastic.max.bulk.size":"2500500","elastic.port":"9300","encodingdetector.charset.min.confidence":"-1","fetcher.bandwidth.target":"-1","fetcher.bandwidth.target.check.everyNSecs":"30","fetcher.follow.outlinks.depth":"-1","fetcher.follow.outlinks.depth.divisor":"2","fetcher.follow.outlinks.ignore.external":"true","fetcher.follow.outlinks.num.links":"4","fetcher.max.crawl.delay":"30","fetcher.max.exceptions.per.queue":"-1","fetcher.maxNum.threads":"25","fetcher.parse":"false","fetcher.queue.depth.multiplier":"50","fetcher.queue.mode":"byHost","fetcher.server.delay":"5.0","fetcher.server.min.delay":"0.0","fetcher.store.content":"true","fetcher.threads.fetch":"10","fetcher.threads.per.queue":"1","fetcher.threads.timeout.divisor":"2","fetcher.throughput.threshold.check.after":"5","fetcher.throughput.threshold.pages":"-1","fetcher.throughput.threshold.retries":"5","fetcher.timelimit.mins":"-1","fetcher.verbose":"false","file.content.ignored":"true","file.content.limit":"65536","file.crawl.parent":"true","file.crawl.redirect_noncanonical":"true","fs.checkpoint.dir":"${hadoop.tmp.dir}/dfs/namesecondary","fs.checkpoint.edits.dir":"${fs.checkpoint.dir}","fs.checkpoint.period":"3600","fs.checkpoint.size":"67108864","fs.default.name":"file:///","fs.file.impl":"org.apache.hadoop.fs.LocalFileSystem","fs.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","fs.har.impl":"org.apache.hadoop.fs.HarFileSystem","fs.har.impl.disable.cache":"true","fs.hdfs.impl":"org.apache.hadoop.hdfs.DistributedFileSystem","fs.hftp.impl":"org.apache.hadoop.hdfs.HftpFileSystem","fs.hsftp.impl":"org.apache.hadoop.hdfs.HsftpFileSystem","fs.kfs.impl":"org.apache.hadoop.fs.kfs.KosmosFileSystem","fs.ramfs.impl":"org.apache.hadoop.fs.InMemoryFileSystem","fs.s3.block.size":"67108864","fs.s3.buffer.dir":"${hadoop.tmp.dir}/s3","fs.s3.impl":"org.apache.hadoop.fs.s3.S3FileSystem","fs.s3.maxRetries":"4","fs.s3.sleepTimeSeconds":"10","fs.s3n.impl":"org.apache.hadoop.fs.s3native.NativeS3FileSystem","fs.trash.interval":"0","fs.webhdfs.impl":"org.apache.hadoop.hdfs.web.WebHdfsFileSystem","ftp.content.limit":"65536","ftp.follow.talk":"false","ftp.keep.connection":"false","ftp.password":"[email protected]","ftp.server.timeout":"100000","ftp.timeout":"60000","ftp.username":"anonymous","generate.count.mode":"host","generate.max.count":"-1","generate.min.interval":"-1","generate.min.score":"0","generate.update.crawldb":"false","hadoop.http.authentication.kerberos.keytab":"${user.home}/hadoop.keytab","hadoop.http.authentication.kerberos.principal":"HTTP/localhost@LOCALHOST","hadoop.http.authentication.signature.secret.file":"${user.home}/hadoop-http-auth-signature-secret","hadoop.http.authentication.simple.anonymous.allowed":"true","hadoop.http.authentication.token.validity":"36000","hadoop.http.authentication.type":"simple","hadoop.jetty.logs.serve.aliases":"true","hadoop.job.history.user.location":"${hadoop.log.dir}/history/user","hadoop.logfile.count":"10","hadoop.logfile.size":"10000000","hadoop.native.lib":"true","hadoop.relaxed.worker.version.check":"false","hadoop.rpc.socket.factory.class.default":"org.apache.hadoop.net.StandardSocketFactory","hadoop.security.authentication":"simple","hadoop.security.authorization":"false","hadoop.security.group.mapping":"org.apache.hadoop.security.ShellBasedUnixGroupsMapping","hadoop.security.instrumentation.requires.admin":"false","hadoop.security.token.service.use_ip":"true","hadoop.security.uid.cache.secs":"14400","hadoop.security.use-weak-http-crypto":"false","hadoop.skip.worker.version.check":"false","hadoop.tmp.dir":"/tmp/hadoop-${user.name}","hadoop.util.hash.type":"murmur","headings":"h1,h2","headings.multivalued":"false","http.accept":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8","http.accept.language":"en-us,en-gb,en;q=0.7,*;q=0.3","http.agent.version":"Nutch-1.10-SNAPSHOT","http.auth.file":"httpclient-auth.xml","http.content.limit":"65536","http.enable.if.modified.since.header":"true","http.max.delays":"100","http.redirect.max":"0","http.robots.403.allow":"true","http.store.responsetime":"true","http.timeout":"10000","http.useHttp11":"false","http.verbose":"false","index.geoip.usage":"insightsService","index.parse.md":"metatag.description,metatag.keywords","indexer.add.domain":"false","indexer.max.content.length":"-1","indexer.max.title.length":"100","indexer.score.power":"0.5","indexer.skip.notmodified":"false","io.bytes.per.checksum":"512","io.compression.codecs":"org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.SnappyCodec","io.file.buffer.size":"4096","io.mapfile.bloom.error.rate":"0.005","io.mapfile.bloom.size":"1048576","io.seqfile.compress.blocksize":"1000000","io.seqfile.lazydecompress":"true","io.seqfile.sorter.recordlimit":"1000000","io.serializations":"org.apache.hadoop.io.serializer.WritableSerialization","io.skip.checksum.errors":"false","ipc.client.connect.max.retries":"10","ipc.client.connection.maxidletime":"10000","ipc.client.idlethreshold":"4000","ipc.client.kill.max":"10","ipc.client.tcpnodelay":"false","ipc.server.listen.queue.size":"128","ipc.server.tcpnodelay":"false","lang.analyze.max.length":"2048","lang.extraction.policy":"detect,identify","lang.identification.only.certain":"false","link.analyze.damping.factor":"0.85f","link.analyze.initial.score":"1.0f","link.analyze.num.iterations":"10","link.delete.gone":"false","link.ignore.internal.domain":"true","link.ignore.internal.host":"true","link.ignore.limit.domain":"true","link.ignore.limit.page":"true","link.loops.depth":"2","link.score.updater.clear.score":"0.0f","local.cache.size":"10737418240","mapreduce.fileoutputcommitter.marksuccessfuljobs":"false","metatags.names":"description,keywords","mime.type.magic":"true","mimetype.filter.file":"mimetype-filter.txt","moreIndexingFilter.indexMimeTypeParts":"true","moreIndexingFilter.mapMimeTypes":"false","net.topology.impl":"org.apache.hadoop.net.NetworkTopology","nutch.conf.uuid":"162fbf70-6ee9-4279-a042-810062d86b49","parse.filter.urls":"true","parse.normalize.urls":"true","parse.plugin.file":"parse-plugins.xml","parser.caching.forbidden.policy":"content","parser.character.encoding.default":"windows-1252","parser.html.form.use_action":"false","parser.html.impl":"neko","parser.skip.truncated":"true","parser.timeout":"30","partition.url.mode":"byHost","plugin.auto-activation":"true","plugin.folders":"plugins","plugin.includes":"protocol-http|urlfilter-regex|parse-(html|tika)|index-(basic|anchor)|indexer-solr|scoring-opic|urlnormalizer-(pass|regex|basic)","scoring.depth.max":"1000","solr.auth":"false","solr.commit.index":"true","solr.commit.size":"250","solr.mapping.file":"solrindex-mapping.xml","store.ip.address":"false","subcollection.default.field":"subcollection","tika.uppercase.element.names":"true","topology.node.switch.mapping.impl":"org.apache.hadoop.net.ScriptBasedMapping","topology.script.number.args":"100","urlfilter.automaton.file":"automaton-urlfilter.txt","urlfilter.domain.file":"domain-urlfilter.txt","urlfilter.prefix.file":"prefix-urlfilter.txt","urlfilter.regex.file":"regex-urlfilter.txt","urlfilter.suffix.file":"suffix-urlfilter.txt","urlnormalizer.loop.count":"1","urlnormalizer.order":"org.apache.nutch.net.urlnormalizer.basic.BasicURLNormalizer
org.apache.nutch.net.urlnormalizer.regex.RegexURLNormalizer","urlnormalizer.regex.file":"regex-normalize.xml","webinterface.private.actions":"false"}[chipotle:~]
mattmann%
{noformat}
Will go ahead and commit this. One sub-issue is that it would be nice to pretty
print the JSON. I think I'll add that in a sub issue.
> Configuration endpoint for 1x REST API [A sub-issue of NUTCH-1931]
> ------------------------------------------------------------------
>
> Key: NUTCH-1966
> URL: https://issues.apache.org/jira/browse/NUTCH-1966
> Project: Nutch
> Issue Type: Sub-task
> Affects Versions: 1.9
> Reporter: Sujen Shah
> Assignee: Chris A. Mattmann
> Priority: Minor
> Labels: REST
>
> The current work going on towards developing a REST service for the Nutch 1x
> branch includes many endpoints. This issue deals with the Configuration
> endpoint of the REST service.
> This is a sub issue for the major issue
> https://issues.apache.org/jira/browse/NUTCH-1931.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
