Author: shalin
Date: Tue Feb 3 20:33:20 2009
New Revision: 740423
URL: http://svn.apache.org/viewvc?rev=740423&view=rev
Log:
SOLR-1000 -- FileListEntityProcessor should not apply fileName filter to
directory names. Wow, we reached 1000 issues!
Modified:
lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FileListEntityProcessor.java
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java
Modified: lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt?rev=740423&r1=740422&r2=740423&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt Tue Feb 3 20:33:20
2009
@@ -109,6 +109,9 @@
14. SOLR-999: XPathRecordReader fails on XMLs with nodes mixed with CDATA
content.
(Fergus McMenemie, Noble Paul via shalin)
+15.SOLR-1000: FileListEntityProcessor should not apply fileName filter to
directory names.
+ (Fergus McMenemie via shalin)
+
Documentation
----------------------
Modified:
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FileListEntityProcessor.java
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FileListEntityProcessor.java?rev=740423&r1=740422&r2=740423&view=diff
==============================================================================
---
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FileListEntityProcessor.java
(original)
+++
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FileListEntityProcessor.java
Tue Feb 3 20:33:20 2009
@@ -85,11 +85,10 @@
if (r != null)
recursive = Boolean.parseBoolean(r);
excludes = context.getEntityAttribute(EXCLUDES);
- if (excludes != null)
+ if (excludes != null) {
excludes = resolver.replaceTokens(excludes);
- if (excludes != null)
excludesPattern = Pattern.compile(excludes);
-
+ }
}
private Date getDate(String dateStr) {
@@ -148,20 +147,23 @@
}
}
- private void getFolderFiles(File dir,
- final List<Map<String, Object>> fileDetails) {
+ private void getFolderFiles(File dir, final List<Map<String, Object>>
fileDetails) {
+ // Fetch an array of file objects that pass the filter, however the
+ // returned array is never populated; accept() always returns false.
+ // Rather we make use of the fileDetails array which is populated as
+ // a side affect of the accept method.
dir.list(new FilenameFilter() {
public boolean accept(File dir, String name) {
- if (fileNamePattern == null) {
+ File fileObj = new File(dir, name);
+ if (fileObj.isDirectory()) {
+ if (recursive) getFolderFiles(fileObj, fileDetails);
+ } else if (fileNamePattern == null) {
addDetails(fileDetails, dir, name);
- return false;
- }
- if (fileNamePattern.matcher(name).find()) {
+ } else if (fileNamePattern.matcher(name).find()) {
if (excludesPattern != null && excludesPattern.matcher(name).find())
return false;
addDetails(fileDetails, dir, name);
}
-
return false;
}
});
@@ -170,12 +172,7 @@
private void addDetails(List<Map<String, Object>> files, File dir, String
name) {
Map<String, Object> details = new HashMap<String, Object>();
File aFile = new File(dir, name);
- if (aFile.isDirectory()) {
- if (!recursive)
- return;
- getFolderFiles(aFile, files);
- return;
- }
+ if (aFile.isDirectory()) return;
long sz = aFile.length();
Date lastModified = new Date(aFile.lastModified());
if (biggerThan != -1 && sz <= biggerThan)
Modified:
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java?rev=740423&r1=740422&r2=740423&view=diff
==============================================================================
---
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java
(original)
+++
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java
Tue Feb 3 20:33:20 2009
@@ -91,7 +91,7 @@
System.out.println("List of files when given OLDER_THAN -- " + fList);
Assert.assertEquals(2, fList.size());
attrs = AbstractDataImportHandlerTest.createMap(
- FileListEntityProcessor.FILE_NAME, "xml$",
+ FileListEntityProcessor.FILE_NAME, ".xml$",
FileListEntityProcessor.BASE_DIR, tmpdir.getAbsolutePath(),
FileListEntityProcessor.NEWER_THAN, "'NOW-2HOURS'");
c = AbstractDataImportHandlerTest.getContext(null,
@@ -108,6 +108,39 @@
Assert.assertEquals(2, fList.size());
}
+ @Test
+ public void testRECURSION() throws IOException {
+ long time = System.currentTimeMillis();
+ File tmpdir = new File("." + time);
+ tmpdir.mkdir();
+ tmpdir.deleteOnExit();
+ File childdir = new File(tmpdir + "/child" );
+ childdir.mkdirs();
+ childdir.deleteOnExit();
+ createFile(childdir, "a.xml", "a.xml".getBytes(), true);
+ createFile(childdir, "b.xml", "b.xml".getBytes(), true);
+ createFile(childdir, "c.props", "c.props".getBytes(), true);
+ Map attrs = AbstractDataImportHandlerTest.createMap(
+ FileListEntityProcessor.FILE_NAME, "^.*\\.xml$",
+ FileListEntityProcessor.BASE_DIR, childdir.getAbsolutePath(),
+ FileListEntityProcessor.RECURSIVE, "true");
+ Context c = AbstractDataImportHandlerTest.getContext(null,
+ new VariableResolverImpl(), null, 0, Collections.EMPTY_LIST,
attrs);
+ FileListEntityProcessor fileListEntityProcessor = new
FileListEntityProcessor();
+ fileListEntityProcessor.init(c);
+ List<String> fList = new ArrayList<String>();
+ while (true) {
+ // Add the documents to the index. NextRow() should only
+ // find two filesnames that match the pattern in fileName
+ Map<String, Object> f = fileListEntityProcessor.nextRow();
+ if (f == null)
+ break;
+ fList.add((String) f.get(FileListEntityProcessor.ABSOLUTE_FILE));
+ }
+ System.out.println("List of files indexed -- " + fList);
+ Assert.assertEquals(2, fList.size());
+ }
+
public static File createFile(File tmpdir, String name, byte[] content,
boolean changeModifiedTime) throws IOException
{
File file = new File(tmpdir.getAbsolutePath() + File.separator + name);
