Author: shalin
Date: Wed Sep 30 10:44:10 2009
New Revision: 820237
URL: http://svn.apache.org/viewvc?rev=820237&view=rev
Log:
SOLR-1473 -- Tests for using arbitrary variables and last_index_time in
FileListEntityProcessor date filters
Modified:
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder2.java
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java
Modified:
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder2.java
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder2.java?rev=820237&r1=820236&r2=820237&view=diff
==============================================================================
---
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder2.java
(original)
+++
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder2.java
Wed Sep 30 10:44:10 2009
@@ -25,6 +25,8 @@
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.Date;
+import java.io.File;
/**
* <p>
@@ -228,6 +230,29 @@
assertQ(req("id:3"), "//*...@numfound='1']");
}
+ @Test
+ public void testFileListEntityProcessor_lastIndexTime() throws Exception {
+ long time = System.currentTimeMillis();
+ File tmpdir = new File("." + time);
+ tmpdir.mkdir();
+ tmpdir.deleteOnExit();
+
+ Map<String, String> params = createMap("baseDir",
tmpdir.getAbsolutePath());
+
+ TestFileListEntityProcessor.createFile(tmpdir, "a.xml",
"a.xml".getBytes(), true);
+ TestFileListEntityProcessor.createFile(tmpdir, "b.xml",
"b.xml".getBytes(), true);
+ TestFileListEntityProcessor.createFile(tmpdir, "c.props",
"c.props".getBytes(), true);
+ super.runFullImport(dataConfigFileList, params);
+ assertQ(req("*:*"), "//*...@numfound='3']");
+
+ // Add a new file after a full index is done
+ TestFileListEntityProcessor.createFile(tmpdir, "t.xml",
"t.xml".getBytes(), false);
+ super.runFullImport(dataConfigFileList, params);
+ // we should find only 1 because by default clean=true is passed
+ // and this particular import should find only one file t.xml
+ assertQ(req("*:*"), "//*...@numfound='1']");
+ }
+
public static class MockTransformer extends Transformer {
public Object transformRow(Map<String, Object> row, Context context) {
Assert.assertTrue("Context gave incorrect data source",
context.getDataSource("mockDs") instanceof MockDataSource2);
@@ -323,4 +348,14 @@
" </entity>\n" +
" </document>\n" +
"</dataConfig>";
+
+ private final String dataConfigFileList = "<dataConfig>\n" +
+ "\t<document>\n" +
+ "\t\t<entity name=\"x\" processor=\"FileListEntityProcessor\" \n" +
+ "\t\t\t\tfileName=\".*\" newerThan=\"${dih.last_index_time}\" \n" +
+ "\t\t\t\tbaseDir=\"${dih.request.baseDir}\"
transformer=\"TemplateTransformer\">\n" +
+ "\t\t\t<field column=\"id\" template=\"${x.file}\" />\n" +
+ "\t\t</entity>\n" +
+ "\t</document>\n" +
+ "</dataConfig>";
}
Modified:
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java?rev=820237&r1=820236&r2=820237&view=diff
==============================================================================
---
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java
(original)
+++
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java
Wed Sep 30 10:44:10 2009
@@ -116,7 +116,7 @@
}
@SuppressWarnings("unchecked")
- private List<String> getFiles(VariableResolverImpl resolver, Map attrs) {
+ static List<String> getFiles(VariableResolverImpl resolver, Map attrs) {
Context c = AbstractDataImportHandlerTest.getContext(null,
resolver, null, Context.FULL_DUMP, Collections.EMPTY_LIST, attrs);
FileListEntityProcessor fileListEntityProcessor = new
FileListEntityProcessor();
@@ -152,6 +152,19 @@
FileListEntityProcessor.NEWER_THAN, "'NOW-2HOURS'");
fList = getFiles(null, attrs);
Assert.assertEquals(2, fList.size());
+
+ // Use a variable for newerThan
+ attrs = AbstractDataImportHandlerTest.createMap(
+ FileListEntityProcessor.FILE_NAME, ".xml$",
+ FileListEntityProcessor.BASE_DIR, tmpdir.getAbsolutePath(),
+ FileListEntityProcessor.NEWER_THAN, "${a.x}");
+ VariableResolverImpl resolver = new VariableResolverImpl();
+ String lastMod = DataImporter.DATE_TIME_FORMAT.get().format(new
Date(System.currentTimeMillis() - 50000));
+ resolver.addNamespace("a", AbstractDataImportHandlerTest.createMap("x",
lastMod));
+ createFile(tmpdir, "t.xml", "t.xml".getBytes(), false);
+ fList = getFiles(resolver, attrs);
+ Assert.assertEquals(1, fList.size());
+ Assert.assertEquals("File name must be t.xml", new File(tmpdir,
"t.xml").getAbsolutePath(), fList.get(0));
}
@Test
