Author: kwright
Date: Wed Jan 8 23:24:46 2014
New Revision: 1556668
URL: http://svn.apache.org/r1556668
Log:
Fix numerous problems with interpreting, packing, and unpacking new value
Modified:
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
Modified:
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java?rev=1556668&r1=1556667&r2=1556668&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
(original)
+++
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
Wed Jan 8 23:24:46 2014
@@ -481,8 +481,8 @@ public class HttpPoster
* @throws ManifoldCFException, ServiceInterruption
*/
public boolean indexPost(String documentURI,
- RepositoryDocument document, Map arguments,
Map<String, List<String>> sourceTargets,
- Boolean keepAllMetadata, String
authorityNameString, IOutputAddActivity activities)
+ RepositoryDocument document, Map arguments, Map<String, List<String>>
sourceTargets,
+ boolean keepAllMetadata, String authorityNameString, IOutputAddActivity
activities)
throws ManifoldCFException, ServiceInterruption
{
if (Logging.ingest.isDebugEnabled())
@@ -754,16 +754,16 @@ public class HttpPoster
*/
protected class IngestThread extends java.lang.Thread
{
- protected String documentURI;
- protected RepositoryDocument document;
- protected Map<String,List<String>> arguments;
- protected Map<String,List<String>> sourceTargets;
- protected String[] shareAcls;
- protected String[] shareDenyAcls;
- protected String[] acls;
- protected String[] denyAcls;
- protected String commitWithin;
- protected Boolean keepAllMetadata;
+ protected final String documentURI;
+ protected final RepositoryDocument document;
+ protected final Map<String,List<String>> arguments;
+ protected final Map<String,List<String>> sourceTargets;
+ protected final String[] shareAcls;
+ protected final String[] shareDenyAcls;
+ protected final String[] acls;
+ protected final String[] denyAcls;
+ protected final String commitWithin;
+ protected final boolean keepAllMetadata;
protected Long activityStart = null;
protected Long activityBytes = null;
@@ -773,14 +773,9 @@ public class HttpPoster
protected boolean readFromDocumentStreamYet = false;
protected boolean rval = false;
- public IngestThread(String documentURI)
- {
- super();}
-
-
public IngestThread(String documentURI, RepositoryDocument document,
- Map<String, List<String>> arguments, Boolean
keepAllMetadata, Map<String, List<String>> sourceTargets,
- String[] shareAcls, String[] shareDenyAcls, String[]
acls, String[] denyAcls, String commitWithin)
+ Map<String, List<String>> arguments, boolean keepAllMetadata,
Map<String, List<String>> sourceTargets,
+ String[] shareAcls, String[] shareDenyAcls, String[] acls, String[]
denyAcls, String commitWithin)
{
super();
setDaemon(true);
@@ -796,15 +791,7 @@ public class HttpPoster
this.keepAllMetadata=keepAllMetadata;
}
- public Boolean getKeepAllMetadata() {
- return keepAllMetadata;
- }
-
- public void setKeepAllMetadata(Boolean keepAllMetadata) {
- this.keepAllMetadata = keepAllMetadata;
- }
-
- public void run()
+ public void run()
{
long length = document.getBinaryLength();
InputStream is = document.getBinaryStream();
@@ -962,50 +949,53 @@ public class HttpPoster
}
}
- /**
- * builds the solr parameter maps for the update request.
- * For each mapping expressed is applied the renaming for the metadata
field name.
- * If we set to keep all the metadata, the metadata non present in the
mapping will be kept with their original names.
- * In the other case ignored
- * @param out
- * @throws IOException
- */
- private void buildSolrParamsFromMetadata(ModifiableSolrParams out)
throws IOException
- {
- if (this.keepAllMetadata)
- {
- Iterator<String> iter = document.getFields();
- while (iter.hasNext())
- {
- String fieldName = iter.next();
- List<String> mappings = sourceTargets.get(fieldName);
- if (mappings != null)
- for (String newFieldName : mappings)
- applySingleMapping(fieldName, out, newFieldName);
- else // the fields not mentioned in the mapping are added
only if we have set the keep all metadata=true.
- this.applySingleMapping(fieldName, out, fieldName);
- }
- }
- else //don't keep all the metadata but only the ones in
sourceTargets
- for (String originalFieldName : sourceTargets.keySet())
- {
- List<String> mapping =
sourceTargets.get(originalFieldName);
- for (String newFieldName : mapping)
- applySingleMapping(originalFieldName, out,
newFieldName);
- }
- }
-
- private void applySingleMapping(String originalFieldName,
ModifiableSolrParams out, String newFieldName) throws IOException {
- if(newFieldName != null && !newFieldName.isEmpty()) {
- if
(newFieldName.toLowerCase(Locale.ROOT).equals(idAttributeName.toLowerCase(Locale.ROOT)))
{
- newFieldName = ID_METADATA;
- }
- String[] values = document.getFieldAsStrings(originalFieldName);
- writeField(out,LITERAL+newFieldName,values);
- }
+ /**
+ * builds the solr parameter maps for the update request.
+ * For each mapping expressed is applied the renaming for the metadata
field name.
+ * If we set to keep all the metadata, the metadata non present in the
mapping will be kept with their original names.
+ * In the other case ignored
+ * @param out
+ * @throws IOException
+ */
+ private void buildSolrParamsFromMetadata(ModifiableSolrParams out) throws
IOException
+ {
+ if (this.keepAllMetadata)
+ {
+ Iterator<String> iter = document.getFields();
+ while (iter.hasNext())
+ {
+ String fieldName = iter.next();
+ List<String> mappings = sourceTargets.get(fieldName);
+ if (mappings != null)
+ for (String newFieldName : mappings)
+ applySingleMapping(fieldName, out, newFieldName);
+ else // the fields not mentioned in the mapping are added only if we
have set the keep all metadata=true.
+ applySingleMapping(fieldName, out, fieldName);
+ }
+ }
+ else
+ {
+ //don't keep all the metadata but only the ones in sourceTargets
+ for (String originalFieldName : sourceTargets.keySet())
+ {
+ List<String> mapping = sourceTargets.get(originalFieldName);
+ for (String newFieldName : mapping)
+ applySingleMapping(originalFieldName, out, newFieldName);
+ }
}
+ }
+
+ private void applySingleMapping(String originalFieldName,
ModifiableSolrParams out, String newFieldName) throws IOException {
+ if(newFieldName != null && !newFieldName.isEmpty()) {
+ if
(newFieldName.toLowerCase(Locale.ROOT).equals(idAttributeName.toLowerCase(Locale.ROOT)))
{
+ newFieldName = ID_METADATA;
+ }
+ String[] values = document.getFieldAsStrings(originalFieldName);
+ writeField(out,LITERAL+newFieldName,values);
+ }
+ }
- public Throwable getException()
+ public Throwable getException()
{
return exception;
}
Modified:
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java?rev=1556668&r1=1556667&r2=1556668&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
(original)
+++
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
Wed Jan 8 23:24:46 2014
@@ -513,16 +513,14 @@ public class SolrConnector extends org.a
// Do the source/target pairs
i = 0;
Map<String, List<String>> sourceTargets = new HashMap<String,
List<String>>();
- SpecificationNode keepAllMetadataNode = null;
+ boolean keepAllMetadata = true;
while (i < spec.getChildCount()) {
SpecificationNode sn = spec.getChild(i++);
if(sn.getType().equals(SolrConfig.NODE_KEEPMETADATA)) {
- keepAllMetadataNode = sn;
- continue;
- }
-
- if (sn.getType().equals(SolrConfig.NODE_FIELDMAP)) {
+ String value = sn.getAttributeValue(SolrConfig.ATTRIBUTE_VALUE);
+ keepAllMetadata = Boolean.parseBoolean(value);
+ } else if (sn.getType().equals(SolrConfig.NODE_FIELDMAP)) {
String source = sn.getAttributeValue(SolrConfig.ATTRIBUTE_SOURCE);
String target = sn.getAttributeValue(SolrConfig.ATTRIBUTE_TARGET);
@@ -565,6 +563,12 @@ public class SolrConnector extends org.a
packList(sb,sourceTargetsList,'+');
+ // Keep all metadata flag
+ if (keepAllMetadata)
+ sb.append('+');
+ else
+ sb.append('-');
+
// Here, append things which we have no intention of unpacking. This
includes stuff that comes from
// the configuration information, for instance.
@@ -596,17 +600,6 @@ public class SolrConnector extends org.a
sb.append('-');
}
- // Add keepAllMetadata option if specified
- if(keepAllMetadataNode != null) {
- fixedList[0] = SolrConfig.NODE_KEEPMETADATA;
- fixedList[1] =
keepAllMetadataNode.getAttributeValue(SolrConfig.ATTRIBUTE_VALUE);
- StringBuilder pairBuffer = new StringBuilder();
- packFixedList(pairBuffer,fixedList,'=');
- List<String> list = new ArrayList<String>();
- list.add(pairBuffer.toString());
- packList(sb, list,'+');
- }
-
return sb.toString();
}
@@ -663,7 +656,6 @@ public class SolrConnector extends org.a
// Build the argument map we'll send.
Map args = new HashMap();
Map<String, List<String>> sourceTargets = new HashMap<String,
List<String>>();
- Boolean keepAllMetadata = false;
int index = 0;
ArrayList nameValues = new ArrayList();
index = unpackList(nameValues,outputDescription,index,'+');
@@ -705,11 +697,12 @@ public class SolrConnector extends org.a
}
// extract keep all metadata Flag
- String keepAllMetadataFlag=(String)metadataExtraParams.get(0);
- unpackFixedList(fixedBuffer, keepAllMetadataFlag, 0, '=');
- String keepAllMetadataFlagValue = fixedBuffer[1];
- keepAllMetadata=Boolean.parseBoolean(keepAllMetadataFlagValue);
-
+ boolean keepAllMetadata = true;
+ if (index < outputDescription.length())
+ {
+ keepAllMetadata = (outputDescription.charAt(index++) == '+');
+ }
+
// Establish a session
getSession();
