Author: shalin
Date: Wed Sep 30 10:35:23 2009
New Revision: 820235
URL: http://svn.apache.org/viewvc?rev=820235&view=rev
Log:
SOLR-1474 -- Delta-import should run even if last_index_time is not set
Modified:
lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java
Modified: lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt?rev=820235&r1=820234&r2=820235&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt Wed Sep 30 10:35:23
2009
@@ -29,6 +29,10 @@
parent-row at the end of row (end of data). A new method EntityProcessor#close
is added which is called at the end
of import.
+In Solr 1.3, if the last_index_time was not available (first import) and a
delta-import was requested, a full-import
+was run instead. This is no longer the case. In Solr 1.4 delta import is run
with last_index_time as the epoch
+date (January 1, 1970, 00:00:00 GMT) if last_index_time is not available.
+
Detailed Change List
----------------------
@@ -265,6 +269,9 @@
30.SOLR-1450: Jdbc connection properties such as batchSize are not applied if
the driver jar is placed
in solr_home/lib.
(Steve Sun via shalin)
+
+31.SOLR-1474: Delta-import should run even if last_index_time is not set.
+ (shalin)
Documentation
Modified:
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java?rev=820235&r1=820234&r2=820235&view=diff
==============================================================================
---
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java
(original)
+++
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java
Wed Sep 30 10:35:23 2009
@@ -22,6 +22,7 @@
import org.apache.solr.common.util.NamedList;
import java.io.IOException;
+import java.io.File;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -46,6 +47,14 @@
@Override
public void tearDown() throws Exception {
+ // remove dataimport.properties
+ File f = new File("solr/conf/dataimport.properties");
+ log.info("Looking for dataimport.properties at: " + f.getAbsolutePath());
+ if (f.exists()) {
+ log.info("Deleting dataimport.properties");
+ if (!f.delete())
+ log.warn("Could not delete dataimport.properties");
+ }
super.tearDown();
}
Modified:
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java?rev=820235&r1=820234&r2=820235&view=diff
==============================================================================
---
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
(original)
+++
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
Wed Sep 30 10:35:23 2009
@@ -252,14 +252,6 @@
this.indexStartTime = indextStartTime;
}
- Date getLastIndexTime() {
- return lastIndexTime;
- }
-
- void setLastIndexTime(Date lastIndexTime) {
- this.lastIndexTime = lastIndexTime;
- }
-
void store(Object key, Object value) {
store.put(key, value);
}
@@ -387,8 +379,6 @@
return;
}
try {
- Date lastModified = sw.loadIndexStartTime();
- setLastIndexTime(lastModified);
if (FULL_IMPORT_CMD.equals(command) || IMPORT_CMD.equals(command)) {
doFullImport(sw, reqParams);
} else if (command.equals(DELTA_IMPORT_CMD)) {
Modified:
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java?rev=820235&r1=820234&r2=820235&view=diff
==============================================================================
---
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
(original)
+++
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
Wed Sep 30 10:35:23 2009
@@ -27,6 +27,7 @@
import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
+import java.text.ParseException;
/**
* <p> DocBuilder is responsible for creating Solr documents out of the given
configuration. It also maintains
@@ -41,6 +42,8 @@
private static final Logger LOG = LoggerFactory.getLogger(DocBuilder.class);
+ private static final Date EPOCH = new Date(0);
+
DataImporter dataImporter;
private DataConfig.Document document;
@@ -76,24 +79,36 @@
}
public VariableResolverImpl getVariableResolver() {
- VariableResolverImpl resolver = new VariableResolverImpl();
- Map<String, Object> indexerNamespace = new HashMap<String, Object>();
- if (dataImporter.getLastIndexTime() != null)
- indexerNamespace.put(LAST_INDEX_TIME,
-
DataImporter.DATE_TIME_FORMAT.get().format(dataImporter.getLastIndexTime()));
- indexerNamespace.put(INDEX_START_TIME, dataImporter.getIndexStartTime());
- indexerNamespace.put("request", requestParameters.requestParams);
- indexerNamespace.put("functions", functionsNamespace);
- for (DataConfig.Entity entity :
dataImporter.getConfig().document.entities) {
- String key = entity.name + "." + SolrWriter.LAST_INDEX_KEY;
- String lastIndex = persistedProperties.getProperty(key);
- if (lastIndex != null) {
- indexerNamespace.put(key, lastIndex);
- }
- }
- resolver.addNamespace(DataConfig.IMPORTER_NS_SHORT, indexerNamespace);
- resolver.addNamespace(DataConfig.IMPORTER_NS, indexerNamespace);
- return resolver;
+ try {
+ VariableResolverImpl resolver = new VariableResolverImpl();
+ Map<String, Object> indexerNamespace = new HashMap<String, Object>();
+ if (persistedProperties.getProperty(LAST_INDEX_TIME) != null) {
+ indexerNamespace.put(LAST_INDEX_TIME,
+
DataImporter.DATE_TIME_FORMAT.get().parse(persistedProperties.getProperty(LAST_INDEX_TIME)));
+ } else {
+ // set epoch
+ indexerNamespace.put(LAST_INDEX_TIME, EPOCH);
+ }
+ indexerNamespace.put(INDEX_START_TIME, dataImporter.getIndexStartTime());
+ indexerNamespace.put("request", requestParameters.requestParams);
+ indexerNamespace.put("functions", functionsNamespace);
+ for (DataConfig.Entity entity :
dataImporter.getConfig().document.entities) {
+ String key = entity.name + "." + SolrWriter.LAST_INDEX_KEY;
+ String lastIndex = persistedProperties.getProperty(key);
+ if (lastIndex != null) {
+ indexerNamespace.put(key, lastIndex);
+ } else {
+ indexerNamespace.put(key, EPOCH);
+ }
+ }
+ resolver.addNamespace(DataConfig.IMPORTER_NS_SHORT, indexerNamespace);
+ resolver.addNamespace(DataConfig.IMPORTER_NS, indexerNamespace);
+ return resolver;
+ } catch (ParseException e) {
+
DataImportHandlerException.wrapAndThrow(DataImportHandlerException.SEVERE, e);
+ // unreachable statement
+ return null;
+ }
}
private void invokeEventListener(String className) {
@@ -153,8 +168,7 @@
DataImporter.DATE_TIME_FORMAT.get().format(new Date()));
root = e;
String delQuery = e.allAttributes.get("preImportDeleteQuery");
- if (dataImporter.getStatus() == DataImporter.Status.RUNNING_DELTA_DUMP
- && dataImporter.getLastIndexTime() != null) {
+ if (dataImporter.getStatus() == DataImporter.Status.RUNNING_DELTA_DUMP) {
cleanByQuery(delQuery, fullCleanDone);
doDelta();
delQuery = e.allAttributes.get("postImportDeleteQuery");
Modified:
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java?rev=820235&r1=820234&r2=820235&view=diff
==============================================================================
---
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java
(original)
+++
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java
Wed Sep 30 10:35:23 2009
@@ -219,22 +219,6 @@
}
}
- public Date loadIndexStartTime() {
- Properties props;
- props = readIndexerProperties();
- String result = props.getProperty(SolrWriter.LAST_INDEX_KEY);
-
- try {
- if (result != null)
- return DataImporter.DATE_TIME_FORMAT.get().parse(result);
- } catch (ParseException e) {
- throw new DataImportHandlerException(DataImportHandlerException.WARN,
- "Unable to read last indexed time from: "
- + persistFilename, e);
- }
- return null;
- }
-
public DebugLogger getDebugLogger() {
if (debugLogger == null) {
debugLogger = new DebugLogger(this);
