Author: shalin
Date: Fri Dec 5 11:14:11 2008
New Revision: 723824
URL: http://svn.apache.org/viewvc?rev=723824&view=rev
Log:
SOLR-893 -- Unable to delete documents via SQL and deletedPkQuery with
deltaimport
Modified:
lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessor2.java
Modified: lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt?rev=723824&r1=723823&r2=723824&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt Fri Dec 5 11:14:11
2008
@@ -63,6 +63,9 @@
8. SOLR-873: Fix case-sensitive field names and columns (Jon Baer, shalin)
+9. SOLR-893: Unable to delete documents via SQL and deletedPkQuery with
deltaimport
+ (Dan Rosher via shalin)
+
Documentation
----------------------
Modified:
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java?rev=723824&r1=723823&r2=723824&view=diff
==============================================================================
---
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
(original)
+++
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
Fri Dec 5 11:14:11 2008
@@ -463,10 +463,8 @@
Set<Map<String, Object>> deltaSet = new HashSet<Map<String, Object>>();
resolver.addNamespace(null, (Map) entity.allAttributes);
EntityProcessor entityProcessor = getEntityProcessor(entity,
context.getCore());
- entityProcessor.init(new ContextImpl(entity, resolver, null,
- Context.FIND_DELTA, session, null, this));
+ entityProcessor.init(new ContextImpl(entity, resolver, null,
Context.FIND_DELTA, session, null, this));
LOG.info("Running ModifiedRowKey() for Entity: " + entity.name);
- int count = 0;
//get the modified rows in this entity
while (true) {
Map<String, Object> row = entityProcessor.nextModifiedRowKey();
@@ -475,27 +473,32 @@
break;
deltaSet.add(row);
- count++;
importStatistics.rowsCount.incrementAndGet();
}
- LOG.info("Completed ModifiedRowKey for Entity: " + entity.name
- + " rows obtained : " + count);
- count = 0;
- // identifying the deleted rows from this entities
- LOG.info("Running DeletedRowKey() for Entity: " + entity.name);
//get the deleted rows for this entity
Set<Map<String, Object>> deletedSet = new HashSet<Map<String, Object>>();
+ Set<Map<String, Object>> deltaRemoveSet = new HashSet<Map<String,
Object>>();
while (true) {
Map<String, Object> row = entityProcessor.nextDeletedRowKey();
if (row == null)
break;
+ //Check to see if this delete is in the current delta set
+ for (Map<String, Object> modifiedRow : deltaSet) {
+ if (modifiedRow.get(entity.pk).equals(row.get(entity.pk))) {
+ deltaRemoveSet.add(modifiedRow);
+ }
+ }
+
deletedSet.add(row);
- count++;
importStatistics.rowsCount.incrementAndGet();
}
- LOG.info("Completed DeletedRowKey for Entity: " + entity.name
- + " rows obtained : " + count);
+
+ //asymmetric Set difference
+ deltaSet.removeAll(deltaRemoveSet);
+
+ LOG.info("Completed ModifiedRowKey for Entity: " + entity.name + " rows
obtained : " + deltaSet.size());
+ LOG.info("Completed DeletedRowKey for Entity: " + entity.name + " rows
obtained : " + deletedSet.size());
myModifiedPks.addAll(deltaSet);
Set<Map<String, Object>> parentKeyList = new HashSet<Map<String,
Object>>();
@@ -507,12 +510,10 @@
// identifying deleted rows with deltas
for (Map<String, Object> row : myModifiedPks)
- getModifiedParentRows(resolver.addNamespace(entity.name, row),
- entity.name, parentEntityProcessor, parentKeyList);
+ getModifiedParentRows(resolver.addNamespace(entity.name, row),
entity.name, parentEntityProcessor, parentKeyList);
// running the same for deletedrows
for (Map<String, Object> row : deletedSet) {
- getModifiedParentRows(resolver.addNamespace(entity.name, row),
- entity.name, parentEntityProcessor, parentKeyList);
+ getModifiedParentRows(resolver.addNamespace(entity.name, row),
entity.name, parentEntityProcessor, parentKeyList);
}
}
LOG.info("Completed parentDeltaQuery for Entity: " + entity.name);
Modified:
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java?rev=723824&r1=723823&r2=723824&view=diff
==============================================================================
---
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java
(original)
+++
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java
Fri Dec 5 11:14:11 2008
@@ -230,10 +230,12 @@
if (rowIterator.hasNext())
return rowIterator.next();
query = null;
+ rowIterator = null;
return null;
} catch (Exception e) {
log.error("getNext() failed for query '" + query + "'", e);
query = null;
+ rowIterator = null;
wrapAndThrow(DataImportHandlerException.WARN, e);
return null;
}
Modified:
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessor2.java
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessor2.java?rev=723824&r1=723823&r2=723824&view=diff
==============================================================================
---
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessor2.java
(original)
+++
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessor2.java
Fri Dec 5 11:14:11 2008
@@ -80,7 +80,7 @@
List parentRow = new ArrayList();
parentRow.add(createMap("id", "5"));
- MockDataSource.setIterator("select * from x where x.id = '5'", parentRow
+ MockDataSource.setIterator("select * from x where id = '5'", parentRow
.iterator());
List childRow = new ArrayList();
@@ -96,6 +96,56 @@
@Test
@SuppressWarnings("unchecked")
+ public void testCompositePk_DeltaImport_DeletedPkQuery() throws Exception {
+ List parentRow = new ArrayList();
+ parentRow.add(createMap("id", "11"));
+ MockDataSource.setIterator("select * from x", parentRow.iterator());
+
+ List childRow = new ArrayList();
+ childRow.add(createMap("desc", "hello"));
+
+ MockDataSource.setIterator("select * from y where y.A=11", childRow
+ .iterator());
+
+ super.runFullImport(dataConfig);
+
+ assertQ(req("id:11"), "//[EMAIL PROTECTED]'1']");
+
+
+
+ List deltaRow = new ArrayList();
+ deltaRow.add(createMap("id", "15"));
+ deltaRow.add(createMap("id", "17"));
+ MockDataSource.setIterator("select id from x where last_modified > NOW",
+ deltaRow.iterator());
+
+ List deltaDeleteRow = new ArrayList();
+ deltaDeleteRow.add(createMap("id", "11"));
+ deltaDeleteRow.add(createMap("id", "17"));
+ MockDataSource.setIterator("select id from x where last_modified > NOW AND
deleted='true'",
+ deltaDeleteRow.iterator());
+
+ parentRow = new ArrayList();
+ parentRow.add(createMap("id", "15"));
+ MockDataSource.setIterator("select * from x where id = '15'", parentRow
+ .iterator());
+
+ parentRow = new ArrayList();
+ parentRow.add(createMap("id", "17"));
+ MockDataSource.setIterator("select * from x where id = '17'", parentRow
+ .iterator());
+
+ super.runDeltaImport(dataConfig);
+
+ assertQ(req("id:15"), "//[EMAIL PROTECTED]'1']");
+ assertQ(req("id:11"), "//[EMAIL PROTECTED]'0']");
+ assertQ(req("id:17"), "//[EMAIL PROTECTED]'0']");
+
+
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_DeltaImportQuery() throws Exception {
List deltaRow = new ArrayList();
deltaRow.add(createMap("id", "5"));
@@ -120,7 +170,7 @@
private static String dataConfig = "<dataConfig>\n"
+ " <document>\n"
- + " <entity name=\"x\" pk=\"x.id\" query=\"select *
from x\" deltaQuery=\"select id from x where last_modified > NOW\">\n"
+ + " <entity name=\"x\" pk=\"id\" query=\"select * from
x\" deletedPkQuery=\"select id from x where last_modified > NOW AND
deleted='true'\" deltaQuery=\"select id from x where last_modified > NOW\">\n"
+ " <field column=\"id\" />\n"
+ " <entity name=\"y\" query=\"select * from y
where y.A=${x.id}\">\n"
+ " <field column=\"desc\" />\n"
