Author: ab Date: Tue Feb 3 15:12:48 2009 New Revision: 740318 URL: http://svn.apache.org/viewvc?rev=740318&view=rev Log: NUTCH-279 Additions to urlnormalizer-regex (modified).
Added: lucene/nutch/trunk/src/java/org/apache/nutch/net/URLNormalizerChecker.java (with props) Modified: lucene/nutch/trunk/CHANGES.txt Modified: lucene/nutch/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/CHANGES.txt?rev=740318&r1=740317&r2=740318&view=diff ============================================================================== --- lucene/nutch/trunk/CHANGES.txt (original) +++ lucene/nutch/trunk/CHANGES.txt Tue Feb 3 15:12:48 2009 @@ -328,6 +328,9 @@ 122. NUTCH-682 - SOLR indexer does not set boost on the document. (julien nioche via dogacan) + +123. NUTCH-279 - Additions to urlnormalizer-regex (Stefan Neufeind, ab) + Release 0.9 - 2007-04-02 Added: lucene/nutch/trunk/src/java/org/apache/nutch/net/URLNormalizerChecker.java URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/net/URLNormalizerChecker.java?rev=740318&view=auto ============================================================================== --- lucene/nutch/trunk/src/java/org/apache/nutch/net/URLNormalizerChecker.java (added) +++ lucene/nutch/trunk/src/java/org/apache/nutch/net/URLNormalizerChecker.java Tue Feb 3 15:12:48 2009 @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nutch.net; + +import org.apache.nutch.plugin.Extension; +import org.apache.nutch.plugin.ExtensionPoint; +import org.apache.nutch.plugin.PluginRepository; + +import org.apache.hadoop.conf.Configuration; + +import org.apache.nutch.util.NutchConfiguration; + +import java.io.BufferedReader; +import java.io.InputStreamReader; + +/** + * Checks one given normalizer or all normalizers. + */ +public class URLNormalizerChecker { + + private Configuration conf; + + public URLNormalizerChecker(Configuration conf) { + this.conf = conf; + } + + private void checkOne(String normalizerName, String scope) throws Exception { + URLNormalizer normalizer = null; + + ExtensionPoint point = + PluginRepository.get(conf).getExtensionPoint(URLNormalizer.X_POINT_ID); + + if (point == null) + throw new RuntimeException(URLNormalizer.X_POINT_ID+" not found."); + + Extension[] extensions = point.getExtensions(); + + for (int i = 0; i < extensions.length; i++) { + Extension extension = extensions[i]; + normalizer = (URLNormalizer)extension.getExtensionInstance(); + if (normalizer.getClass().getName().equals(normalizerName)) { + break; + } else { + normalizer = null; + } + } + + if (normalizer == null) + throw new RuntimeException("URLNormalizer "+normalizerName+" not found."); + + System.out.println("Checking URLNormalizer " + normalizerName); + + BufferedReader in = new BufferedReader(new InputStreamReader(System.in)); + String line; + while ((line = in.readLine()) != null) { + String out = normalizer.normalize(line, scope); + System.out.println(out); + } + } + + private void checkAll(String scope) throws Exception { + System.out.println("Checking combination of all URLNormalizers available"); + + BufferedReader in = new BufferedReader(new InputStreamReader(System.in)); + String line; + URLNormalizers normalizers = new URLNormalizers(conf, scope); + while((line = in.readLine()) != null) { + String out = normalizers.normalize(line, scope); + System.out.println(out); + } + } + + public static void main(String[] args) throws Exception { + + String usage = "Usage: URLNormalizerChecker [-normalizer <normalizerName>] [-scope <scope>]" + + "\n\tscope can be one of: default,partition,generate_host_count,fetcher,crawldb,linkdb,inject,outlink"; + + String normalizerName = null; + String scope = URLNormalizers.SCOPE_DEFAULT; + for (int i = 0; i < args.length; i++) { + if (args[i].equals("-normalizer")) { + normalizerName = args[++i]; + } else if (args[i].equals("-scope")) { + scope = args[++i]; + } else { + System.err.println(usage); + System.exit(-1); + } + } + + URLNormalizerChecker checker = new URLNormalizerChecker(NutchConfiguration.create()); + if (normalizerName != null) { + checker.checkOne(normalizerName, scope); + } else { + checker.checkAll(scope); + } + + System.exit(0); + } +} Propchange: lucene/nutch/trunk/src/java/org/apache/nutch/net/URLNormalizerChecker.java ------------------------------------------------------------------------------ svn:eol-style = native