svn commit: r686900 - in /lucene/nutch/trunk: CHANGES.txt src/java/org/apache/nutch/indexer/IndexSorter.java src/test/org/apache/nutch/indexer/TestIndexSorter.java
Author: ab
Date: Mon Aug 18 16:56:20 2008
New Revision: 686900
URL: http://svn.apache.org/viewvc?rev=686900view=rev
Log:
NUTCH-641 IndexSorter incorrectly copies stored fields.
Added:
lucene/nutch/trunk/src/test/org/apache/nutch/indexer/TestIndexSorter.java
(with props)
Modified:
lucene/nutch/trunk/CHANGES.txt
lucene/nutch/trunk/src/java/org/apache/nutch/indexer/IndexSorter.java
Modified: lucene/nutch/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/CHANGES.txt?rev=686900r1=686899r2=686900view=diff
==
--- lucene/nutch/trunk/CHANGES.txt (original)
+++ lucene/nutch/trunk/CHANGES.txt Mon Aug 18 16:56:20 2008
@@ -256,6 +256,9 @@
93. NUTCH-634 - Upgrade Nutch to Hadoop 0.17.1 (Michael Gottesman, Lincoln
Ritter, ab)
+94. NUTCH-641 - IndexSorter inorrectly copies stored fields (ab)
+
+
Release 0.9 - 2007-04-02
1. Changed log4j confiquration to log to stdout on commandline
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/indexer/IndexSorter.java
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/indexer/IndexSorter.java?rev=686900r1=686899r2=686900view=diff
==
--- lucene/nutch/trunk/src/java/org/apache/nutch/indexer/IndexSorter.java
(original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/indexer/IndexSorter.java Mon
Aug 18 16:56:20 2008
@@ -191,6 +191,11 @@
return super.document(newToOld[n]);
}
+public Document document(int n, FieldSelector fieldSelector)
+throws CorruptIndexException, IOException {
+ return super.document(newToOld[n], fieldSelector);
+}
+
public boolean isDeleted(int n) {
return false;
}
@@ -240,6 +245,10 @@
return this.score that.score ? 1 : -1 ;
}
}
+
+public String toString() {
+ return oldDoc= + oldDoc + ,score= + score;
+}
}
public IndexSorter() {
Added: lucene/nutch/trunk/src/test/org/apache/nutch/indexer/TestIndexSorter.java
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/src/test/org/apache/nutch/indexer/TestIndexSorter.java?rev=686900view=auto
==
--- lucene/nutch/trunk/src/test/org/apache/nutch/indexer/TestIndexSorter.java
(added)
+++ lucene/nutch/trunk/src/test/org/apache/nutch/indexer/TestIndexSorter.java
Mon Aug 18 16:56:20 2008
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the License); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nutch.indexer;
+
+import java.io.File;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Index;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.search.Similarity;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.nutch.analysis.NutchDocumentAnalyzer;
+import org.apache.nutch.util.NutchConfiguration;
+
+import junit.framework.TestCase;
+
+public class TestIndexSorter extends TestCase {
+ private static final Log LOG = LogFactory.getLog(TestIndexSorter.class);
+
+ private static final String INDEX_PLAIN = index;
+ private static final String INDEX_SORTED = index-sorted;
+ private static final int NUM_DOCS = 254;
+ private String[] fieldNames = new String[] {
+ id,
+ url,
+ site,
+ content,
+ host,
+ anchor,
+ boost
+ };
+
+ Configuration conf = null;
+ File testDir = null;
+ Directory dir = null;
+
+
+ protected void setUp() throws Exception {
+if (conf == null) conf = NutchConfiguration.create();
+// create test index
+testDir = new File(indexSorter-test- + System.currentTimeMillis());
+if (!testDir.mkdirs()) {
+ throw new Exception(Can't create test dir + testDir.toString());
+}
+LOG.info(Creating
svn commit: r686910 - in /lucene/nutch/trunk: CHANGES.txt src/plugin/parse-swf/sample/test1.txt src/plugin/parse-swf/sample/test2.txt src/plugin/parse-swf/src/java/org/apache/nutch/parse/swf/SWFParser
Author: ab
Date: Mon Aug 18 17:42:07 2008
New Revision: 686910
URL: http://svn.apache.org/viewvc?rev=686910view=rev
Log:
NUTCH-645 Parse-swf unit test failing - fix.
Modified:
lucene/nutch/trunk/CHANGES.txt
lucene/nutch/trunk/src/plugin/parse-swf/sample/test1.txt
lucene/nutch/trunk/src/plugin/parse-swf/sample/test2.txt
lucene/nutch/trunk/src/plugin/parse-swf/src/java/org/apache/nutch/parse/swf/SWFParser.java
Modified: lucene/nutch/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/CHANGES.txt?rev=686910r1=686909r2=686910view=diff
==
--- lucene/nutch/trunk/CHANGES.txt (original)
+++ lucene/nutch/trunk/CHANGES.txt Mon Aug 18 17:42:07 2008
@@ -258,6 +258,8 @@
94. NUTCH-641 - IndexSorter inorrectly copies stored fields (ab)
+95. NUTCH-645 - Parse-swf unit test failing (ab)
+
Release 0.9 - 2007-04-02
Modified: lucene/nutch/trunk/src/plugin/parse-swf/sample/test1.txt
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-swf/sample/test1.txt?rev=686910r1=686909r2=686910view=diff
==
--- lucene/nutch/trunk/src/plugin/parse-swf/sample/test1.txt (original)
+++ lucene/nutch/trunk/src/plugin/parse-swf/sample/test1.txt Mon Aug 18
17:42:07 2008
@@ -1,60 +1,60 @@
-Help
-javascript:openCrosslinkWindow('/go/adobeacquisition')
-Macromedia Home
+/go/gnav_cart
+/go/gnav_company
+/go/gnav_devnet
+/go/gnav_downloads
+/go/gnav_fl_minmessage
+/go/gnav_help
+/go/gnav_mm_home
+/go/gnav_products
/go/gnav_search?loc=en_us
-MovieClip
-solutions
/go/gnav_showcase
-_sans
-rollOut
-To ensure the best possible Internet Experience, please download the latest
version of the free
+/go/gnav_solutions
/go/gnav_store
+/go/gnav_support
+/go/gnav_your_account
+Acquisition Info
+Adobe Home
+AppleGothic
+Array
+Company
+Developers
+Downloads
+Help
+Home
International
+LocaleManager
+Macromedia Flash Player
+Macromedia Home
+MovieClip
Products
+Showcase
+Solutions
+Store
+String
+Support
+TextFormat
+To ensure the best possible Internet Experience, please download the latest
version of the free
+Verdana
+_sans
+active
+bluePill
+button
+color
+company
devnet
+downloads
en_us
-/go/gnav_products
-AppleGothic
-Macromedia Flash Player
-active
+home
+javascript:openCrosslinkWindow('/go/adobeacquisition')
+javascript:openCrosslinkWindow('/go/gnav_adobe_home')
products
-String
-Store
-downloads
+rollOut
rollOver
-Adobe Home
-/go/gnav_your_account
-/go/gnav_downloads
-Showcase
-bluePill
-/go/gnav_company
-/go/gnav_support
-/go/gnav_help
-javascript:openCrosslinkWindow('/go/gnav_adobe_home')
-home
-Home
-Array
-/go/gnav_fl_minmessage
-textColor
-Developers
-Support
-color
-support
+selected
showcase
-button
-/go/gnav_mm_home
+solutions
+support
tabHolder
-selected
-Solutions
-LocaleManager
-Verdana
-/go/gnav_devnet
-Acquisition Info
-/go/gnav_cart
-Company
-/go/gnav_solutions
-company
-Downloads
-TextFormat
+textColor
Modified: lucene/nutch/trunk/src/plugin/parse-swf/sample/test2.txt
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-swf/sample/test2.txt?rev=686910r1=686909r2=686910view=diff
==
--- lucene/nutch/trunk/src/plugin/parse-swf/sample/test2.txt (original)
+++ lucene/nutch/trunk/src/plugin/parse-swf/sample/test2.txt Mon Aug 18
17:42:07 2008
@@ -1,5 +1,5 @@
Impact Impact Impact Arial Arial Arial Webdings Webdings Webdings Verdana
Verdana Verdana CourierNew CourierNew CourierNew Bimini Bimini Bimini
-font
-color
TextFormat
+color
+font
Modified:
lucene/nutch/trunk/src/plugin/parse-swf/src/java/org/apache/nutch/parse/swf/SWFParser.java
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-swf/src/java/org/apache/nutch/parse/swf/SWFParser.java?rev=686910r1=686909r2=686910view=diff
==
---
lucene/nutch/trunk/src/plugin/parse-swf/src/java/org/apache/nutch/parse/swf/SWFParser.java
(original)
+++
lucene/nutch/trunk/src/plugin/parse-swf/src/java/org/apache/nutch/parse/swf/SWFParser.java
Mon Aug 18 17:42:07 2008
@@ -178,10 +178,11 @@
public String getActionText() {
StringBuffer res = new StringBuffer();
-Iterator it = actionStrings.iterator();
-while (it.hasNext()) {
- if (res.length() 0) res.append('\n');
- res.append(it.next());
+String[] strings = (String[])actionStrings.toArray(new
String[actionStrings.size()]);
+Arrays.sort(strings);
+for (int i = 0; i strings.length; i++) {
+ if (i 0) res.append('\n');
+ res.append(strings[i]);
}
return res.toString();
}
svn commit: r686912 - in /lucene/nutch/trunk: CHANGES.txt build.xml
Author: ab
Date: Mon Aug 18 17:49:45 2008
New Revision: 686912
URL: http://svn.apache.org/viewvc?rev=686912view=rev
Log:
NUTCH-642 - Unit tests fail when run in non-local mode.
Modified:
lucene/nutch/trunk/CHANGES.txt
lucene/nutch/trunk/build.xml
Modified: lucene/nutch/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/CHANGES.txt?rev=686912r1=686911r2=686912view=diff
==
--- lucene/nutch/trunk/CHANGES.txt (original)
+++ lucene/nutch/trunk/CHANGES.txt Mon Aug 18 17:49:45 2008
@@ -260,6 +260,8 @@
95. NUTCH-645 - Parse-swf unit test failing (ab)
+96. NUTCH-642 - Unit tests fail when run in non-local mode (ab)
+
Release 0.9 - 2007-04-02
Modified: lucene/nutch/trunk/build.xml
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/build.xml?rev=686912r1=686911r2=686912view=diff
==
--- lucene/nutch/trunk/build.xml (original)
+++ lucene/nutch/trunk/build.xml Mon Aug 18 17:49:45 2008
@@ -40,6 +40,7 @@
pathelement location=${test.src.dir}/
pathelement location=${plugins.classpath.dir}/
path refid=classpath/
+pathelement location=${build.dir}/${final.name}.job /
/path
!-- xmlcatalog definition for xslt task --
@@ -264,7 +265,7 @@
!-- == --
target name=test depends=test-core, test-plugins/
- target name=test-core depends=compile, compile-core-test
+ target name=test-core depends=job, compile-core-test
delete dir=${test.build.data}/
mkdir dir=${test.build.data}/
