Author: kwright
Date: Wed Jan 8 10:30:53 2014
New Revision: 1556487
URL: http://svn.apache.org/r1556487
Log:
Commit Alessandro's patch
Added:
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/test/java/org/
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/test/java/org/apache/
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/test/java/org/apache/manifoldcf/
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/test/java/org/apache/manifoldcf/agents/
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/test/java/org/apache/manifoldcf/agents/output/
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/test/java/org/apache/manifoldcf/agents/output/solr/
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/test/java/org/apache/manifoldcf/agents/output/solr/HttpPosterTest.java
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/test/java/org/apache/manifoldcf/agents/output/solr/IngestThreadTest.java
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/test/java/org/apache/manifoldcf/agents/output/solr/SolrConnectorTest.java
Modified:
manifoldcf/branches/CONNECTORS-840/build.xml
manifoldcf/branches/CONNECTORS-840/connectors/connector-build.xml
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties
manifoldcf/branches/CONNECTORS-840/connectors/solr/pom.xml
manifoldcf/branches/CONNECTORS-840/pom.xml
Modified: manifoldcf/branches/CONNECTORS-840/build.xml
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-840/build.xml?rev=1556487&r1=1556486&r2=1556487&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-840/build.xml (original)
+++ manifoldcf/branches/CONNECTORS-840/build.xml Wed Jan 8 10:30:53 2014
@@ -3899,6 +3899,26 @@ Use Apache Forrest version forrest-0.9-d
<param name="artifact-type" value="jar"/>
</antcall>
</target>
+
+ <target name="download-powerMockModule">
+ <mkdir dir="lib"/>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="org/powermock"/>
+ <param name="artifact-version" value="1.5.2"/>
+ <param name="artifact-name" value="powermock-module-junit4"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ </target>
+
+ <target name="download-powerMockMockito">
+ <mkdir dir="lib"/>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="org/powermock"/>
+ <param name="artifact-version" value="1.5.2"/>
+ <param name="artifact-name" value="powermock-api-mockito"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ </target>
<target name="download-junit">
<mkdir dir="lib"/>
@@ -3909,7 +3929,7 @@ Use Apache Forrest version forrest-0.9-d
<param name="artifact-type" value="jar"/>
</antcall>
</target>
-
+
<target name="download-log4j">
<mkdir dir="lib"/>
<antcall target="download-via-maven"><param name="target" value="lib"/>
@@ -4248,7 +4268,7 @@ Use Apache Forrest version forrest-0.9-d
</antcall>
</target>
- <target name="make-core-deps"
depends="download-jira-client,download-google-api-client,download-dropbox-client,download-solrj,download-zookeeper,download-httpcomponents,download-json,download-hsqldb,download-xerces,download-commons,download-elasticsearch-plugin,download-solr-plugins,download-sharepoint-plugins,download-jstl,download-xmlgraphics-commons,download-wstx-asl,download-xmlsec,download-xml-apis,download-wss4j,download-velocity,download-streambuffer,download-stax,download-servlet-api,download-xml-resolver,download-osgi,download-opensaml,download-mimepull,download-mail,download-log4j,download-junit,download-jaxws,download-glassfish,download-jaxb,download-tomcat,download-h2,download-h2-support,download-geronimo-specs,download-fop,download-derby,download-postgresql,download-axis,download-saaj,download-wsdl4j,download-castor,download-jetty,download-slf4j,download-xalan,download-activation,download-avalon-framework,download-poi,download-chemistry,download-ecj,download-hadoop"
>
+ <target name="make-core-deps"
depends="download-jira-client,download-google-api-client,download-dropbox-client,download-solrj,download-zookeeper,download-httpcomponents,download-json,download-hsqldb,download-xerces,download-commons,download-elasticsearch-plugin,download-solr-plugins,download-sharepoint-plugins,download-jstl,download-xmlgraphics-commons,download-wstx-asl,download-xmlsec,download-xml-apis,download-wss4j,download-velocity,download-streambuffer,download-stax,download-servlet-api,download-xml-resolver,download-osgi,download-opensaml,download-mimepull,download-mail,download-log4j,download-junit,download-powerMockModule,download-powerMockMockito,download-jaxws,download-glassfish,download-jaxb,download-tomcat,download-h2,download-h2-support,download-geronimo-specs,download-fop,download-derby,download-postgresql,download-axis,download-saaj,download-wsdl4j,download-castor,download-jetty,download-slf4j,download-xalan,download-activation,download-avalon-framework,download-p
oi,download-chemistry,download-ecj,download-hadoop">
<copy todir="lib">
<fileset dir="lib-license" includes="*.txt"/>
</copy>
Modified: manifoldcf/branches/CONNECTORS-840/connectors/connector-build.xml
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-840/connectors/connector-build.xml?rev=1556487&r1=1556486&r2=1556487&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-840/connectors/connector-build.xml (original)
+++ manifoldcf/branches/CONNECTORS-840/connectors/connector-build.xml Wed Jan
8 10:30:53 2014
@@ -48,6 +48,7 @@
<include name="jsp-2.1*.jar"/>
<include name="jsp-api*.jar"/>
<include name="junit*.jar"/>
+ <include name="powermock*.jar"/>
<include name="log4j*.jar"/>
<include name="serializer*.jar"/>
<include name="servlet-api*.jar"/>
Modified:
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java?rev=1556487&r1=1556486&r2=1556487&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
(original)
+++
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
Wed Jan 8 10:30:53 2014
@@ -19,8 +19,6 @@
package org.apache.manifoldcf.agents.output.solr;
import org.apache.manifoldcf.core.interfaces.*;
-import org.apache.manifoldcf.core.common.Base64;
-import org.apache.manifoldcf.core.common.XMLDoc;
import org.apache.manifoldcf.core.common.DateParser;
import org.apache.manifoldcf.agents.interfaces.*;
import org.apache.manifoldcf.agents.system.*;
@@ -28,31 +26,21 @@ import org.apache.manifoldcf.agents.syst
import java.io.*;
import java.net.*;
import java.util.*;
-import javax.net.*;
-import javax.net.ssl.*;
import java.util.regex.*;
-import org.apache.log4j.*;
-
-import java.util.concurrent.TimeUnit;
-
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.Credentials;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.PoolingClientConnectionManager;
-import org.apache.http.params.HttpConnectionParams;
-import org.apache.http.params.HttpParams;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.CoreConnectionPNames;
import org.apache.http.params.CoreProtocolPNames;
import org.apache.http.client.params.ClientPNames;
-import org.apache.http.client.params.HttpClientParams;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.conn.ssl.AllowAllHostnameVerifier;
-import org.apache.http.conn.params.ConnRoutePNames;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.protocol.HttpContext;
@@ -483,17 +471,18 @@ public class HttpPoster
/**
* Post the input stream to ingest
- * @param documentURI is the document's uri.
- * @param document is the document structure to ingest.
- * @param arguments are the configuration arguments to pass in the post. Key
is argument name, value is a list of the argument values.
- * @param authorityNameString is the name of the governing authority for this
document's acls, or null if none.
- * @param activities is the activities object, so we can report what's
happening.
- * @return true if the ingestion was successful, or false if the ingestion is
illegal.
+ *
+ * @param documentURI is the document's uri.
+ * @param document is the document structure to ingest.
+ * @param arguments are the configuration arguments to pass in the post.
Key is argument name, value is a list of the argument values.
+ * @param keepAllMetadata
+ *@param authorityNameString is the name of the governing authority for this
document's acls, or null if none.
+ * @param activities is the activities object, so we can report what's
happening. @return true if the ingestion was successful, or false if the
ingestion is illegal.
* @throws ManifoldCFException, ServiceInterruption
*/
public boolean indexPost(String documentURI,
- RepositoryDocument document, Map arguments, Map<String, List<String>>
sourceTargets,
- String authorityNameString, IOutputAddActivity activities)
+ RepositoryDocument document, Map arguments,
Map<String, List<String>> sourceTargets,
+ Boolean keepAllMetadata, String
authorityNameString, IOutputAddActivity activities)
throws ManifoldCFException, ServiceInterruption
{
if (Logging.ingest.isDebugEnabled())
@@ -515,7 +504,7 @@ public class HttpPoster
try
{
- IngestThread t = new
IngestThread(documentURI,document,arguments,sourceTargets,shareAcls,shareDenyAcls,acls,denyAcls,commitWithin);
+ IngestThread t = new
IngestThread(documentURI,document,arguments,keepAllMetadata,sourceTargets,shareAcls,shareDenyAcls,acls,denyAcls,commitWithin);
try
{
t.start();
@@ -774,6 +763,7 @@ public class HttpPoster
protected String[] acls;
protected String[] denyAcls;
protected String commitWithin;
+ protected Boolean keepAllMetadata;
protected Long activityStart = null;
protected Long activityBytes = null;
@@ -783,9 +773,14 @@ public class HttpPoster
protected boolean readFromDocumentStreamYet = false;
protected boolean rval = false;
+ public IngestThread(String documentURI)
+ {
+ super();}
+
+
public IngestThread(String documentURI, RepositoryDocument document,
- Map<String,List<String>> arguments, Map<String, List<String>>
sourceTargets,
- String[] shareAcls, String[] shareDenyAcls, String[] acls, String[]
denyAcls, String commitWithin)
+ Map<String, List<String>> arguments, Boolean
keepAllMetadata, Map<String, List<String>> sourceTargets,
+ String[] shareAcls, String[] shareDenyAcls, String[]
acls, String[] denyAcls, String commitWithin)
{
super();
setDaemon(true);
@@ -798,9 +793,18 @@ public class HttpPoster
this.denyAcls = denyAcls;
this.sourceTargets = sourceTargets;
this.commitWithin = commitWithin;
+ this.keepAllMetadata=keepAllMetadata;
}
- public void run()
+ public Boolean getKeepAllMetadata() {
+ return keepAllMetadata;
+ }
+
+ public void setKeepAllMetadata(Boolean keepAllMetadata) {
+ this.keepAllMetadata = keepAllMetadata;
+ }
+
+ public void run()
{
long length = document.getBinaryLength();
InputStream is = document.getBinaryStream();
@@ -868,32 +872,7 @@ public class HttpPoster
}
// Write the metadata, each in a field by itself
- Iterator<String> iter = document.getFields();
- while (iter.hasNext())
- {
- String fieldName = iter.next();
- List<String> mapping = sourceTargets.get(fieldName);
- if(mapping != null) {
- for(String newFieldName : mapping) {
- if(newFieldName != null && !newFieldName.isEmpty()) {
- if
(newFieldName.toLowerCase(Locale.ROOT).equals(idAttributeName.toLowerCase(Locale.ROOT)))
{
- newFieldName = ID_METADATA;
- }
- String[] values = document.getFieldAsStrings(fieldName);
- writeField(out,LITERAL+newFieldName,values);
- }
- }
- } else {
- String newFieldName = fieldName;
- if (!newFieldName.isEmpty()) {
- if
(newFieldName.toLowerCase(Locale.ROOT).equals(idAttributeName.toLowerCase(Locale.ROOT)))
{
- newFieldName = ID_METADATA;
- }
- String[] values = document.getFieldAsStrings(fieldName);
- writeField(out,LITERAL+newFieldName,values);
- }
- }
- }
+ buildSolrParamsFromMetadata(out);
// These are unnecessary now in the case of non-solrcloud setups,
because we overrode the SolrJ posting method to use multipart.
//writeField(out,LITERAL+"stream_size",String.valueOf(length));
@@ -983,7 +962,50 @@ public class HttpPoster
}
}
- public Throwable getException()
+ /**
+ * builds the solr parameter maps for the update request.
+ * For each mapping expressed is applied the renaming for the metadata
field name.
+ * If we set to keep all the metadata, the metadata non present in the
mapping will be kept with their original names.
+ * In the other case ignored
+ * @param out
+ * @throws IOException
+ */
+ private void buildSolrParamsFromMetadata(ModifiableSolrParams out)
throws IOException
+ {
+ if (this.keepAllMetadata)
+ {
+ Iterator<String> iter = document.getFields();
+ while (iter.hasNext())
+ {
+ String fieldName = iter.next();
+ List<String> mappings = sourceTargets.get(fieldName);
+ if (mappings != null)
+ for (String newFieldName : mappings)
+ applySingleMapping(fieldName, out, newFieldName);
+ else // the fields not mentioned in the mapping are added
only if we have set the keep all metadata=true.
+ this.applySingleMapping(fieldName, out, fieldName);
+ }
+ }
+ else //don't keep all the metadata but only the ones in
sourceTargets
+ for (String originalFieldName : sourceTargets.keySet())
+ {
+ List<String> mapping =
sourceTargets.get(originalFieldName);
+ for (String newFieldName : mapping)
+ applySingleMapping(originalFieldName, out,
newFieldName);
+ }
+ }
+
+ private void applySingleMapping(String originalFieldName,
ModifiableSolrParams out, String newFieldName) throws IOException {
+ if(newFieldName != null && !newFieldName.isEmpty()) {
+ if
(newFieldName.toLowerCase(Locale.ROOT).equals(idAttributeName.toLowerCase(Locale.ROOT)))
{
+ newFieldName = ID_METADATA;
+ }
+ String[] values = document.getFieldAsStrings(originalFieldName);
+ writeField(out,LITERAL+newFieldName,values);
+ }
+ }
+
+ public Throwable getException()
{
return exception;
}
Modified:
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java?rev=1556487&r1=1556486&r2=1556487&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
(original)
+++
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
Wed Jan 8 10:30:53 2014
@@ -126,5 +126,10 @@ public class SolrConfig
public static final String ATTRIBUTE_SOURCE = "source";
/** Attribute describing a target field name */
public static final String ATTRIBUTE_TARGET = "target";
+
+ /**
+ * Node describing the keep metadata
+ */
+ public static final String NODE_KEEPMETADATA = "keepAllMetadata";
}
Modified:
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java?rev=1556487&r1=1556486&r2=1556487&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
(original)
+++
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
Wed Jan 8 10:30:53 2014
@@ -18,11 +18,31 @@
*/
package org.apache.manifoldcf.agents.output.solr;
-import org.apache.manifoldcf.core.interfaces.*;
-import org.apache.manifoldcf.agents.interfaces.*;
-
-import java.util.*;
-import java.io.*;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import org.apache.manifoldcf.agents.interfaces.IOutputAddActivity;
+import org.apache.manifoldcf.agents.interfaces.IOutputNotifyActivity;
+import org.apache.manifoldcf.agents.interfaces.IOutputRemoveActivity;
+import org.apache.manifoldcf.agents.interfaces.OutputSpecification;
+import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
+import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
+import org.apache.manifoldcf.core.interfaces.ConfigNode;
+import org.apache.manifoldcf.core.interfaces.ConfigParams;
+import org.apache.manifoldcf.core.interfaces.ConfigurationNode;
+import org.apache.manifoldcf.core.interfaces.IDFactory;
+import org.apache.manifoldcf.core.interfaces.IHTTPOutput;
+import org.apache.manifoldcf.core.interfaces.IKeystoreManager;
+import org.apache.manifoldcf.core.interfaces.IPostParameters;
+import org.apache.manifoldcf.core.interfaces.IThreadContext;
+import org.apache.manifoldcf.core.interfaces.KeystoreManagerFactory;
+import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+import org.apache.manifoldcf.core.interfaces.SpecificationNode;
/** This is the output connector for SOLR. Currently, no frills.
@@ -493,11 +513,19 @@ public class SolrConnector extends org.a
// Do the source/target pairs
i = 0;
Map<String, List<String>> sourceTargets = new HashMap<String,
List<String>>();
+ SpecificationNode keepAllMetadataNode = null;
while (i < spec.getChildCount()) {
SpecificationNode sn = spec.getChild(i++);
+
+ if(sn.getType().equals(SolrConfig.NODE_KEEPMETADATA)) {
+ keepAllMetadataNode = sn;
+ continue;
+ }
+
if (sn.getType().equals(SolrConfig.NODE_FIELDMAP)) {
String source = sn.getAttributeValue(SolrConfig.ATTRIBUTE_SOURCE);
String target = sn.getAttributeValue(SolrConfig.ATTRIBUTE_TARGET);
+
if (target == null) {
target = "";
}
@@ -568,6 +596,17 @@ public class SolrConnector extends org.a
sb.append('-');
}
+ // Add keepAllMetadata option if specified
+ if(keepAllMetadataNode != null) {
+ fixedList[0] = SolrConfig.NODE_KEEPMETADATA;
+ fixedList[1] =
keepAllMetadataNode.getAttributeValue(SolrConfig.ATTRIBUTE_VALUE);
+ StringBuilder pairBuffer = new StringBuilder();
+ packFixedList(pairBuffer,fixedList,'=');
+ List<String> list = new ArrayList<String>();
+ list.add(pairBuffer.toString());
+ packList(sb, list,'+');
+ }
+
return sb.toString();
}
@@ -624,11 +663,14 @@ public class SolrConnector extends org.a
// Build the argument map we'll send.
Map args = new HashMap();
Map<String, List<String>> sourceTargets = new HashMap<String,
List<String>>();
+ Boolean keepAllMetadata = false;
int index = 0;
ArrayList nameValues = new ArrayList();
index = unpackList(nameValues,outputDescription,index,'+');
ArrayList sts = new ArrayList();
index = unpackList(sts,outputDescription,index,'+');
+ ArrayList metadataExtraParams = new ArrayList();
+ index = unpackList(metadataExtraParams, outputDescription, index, '+');
String[] fixedBuffer = new String[2];
// Do the name/value pairs
@@ -662,11 +704,17 @@ public class SolrConnector extends org.a
list.add(target);
}
+ // extract keep all metadata Flag
+ String keepAllMetadataFlag=(String)metadataExtraParams.get(0);
+ unpackFixedList(fixedBuffer, keepAllMetadataFlag, 0, '=');
+ String keepAllMetadataFlagValue = fixedBuffer[1];
+ keepAllMetadata=Boolean.parseBoolean(keepAllMetadataFlagValue);
+
// Establish a session
getSession();
// Now, go off and call the ingest API.
- if
(poster.indexPost(documentURI,document,args,sourceTargets,authorityNameString,activities))
+ if
(poster.indexPost(documentURI,document,args,sourceTargets,keepAllMetadata,authorityNameString,activities))
return DOCUMENTSTATUS_ACCEPTED;
return DOCUMENTSTATUS_REJECTED;
}
@@ -2289,6 +2337,7 @@ public class SolrConnector extends org.a
int fieldCounter = 0;
i = 0;
+ SpecificationNode keepMetadata = null;
while (i < os.getChildCount()) {
SpecificationNode sn = os.getChild(i++);
if (sn.getType().equals(SolrConfig.NODE_FIELDMAP)) {
@@ -2325,6 +2374,9 @@ public class SolrConnector extends org.a
);
fieldCounter++;
}
+ else if(sn.getType().equals(SolrConfig.NODE_KEEPMETADATA)) {
+ keepMetadata = sn;
+ }
}
if (fieldCounter == 0)
@@ -2333,6 +2385,12 @@ public class SolrConnector extends org.a
" <tr class=\"formrow\"><td class=\"formmessage\" colspan=\"3\">" +
Messages.getBodyString(locale,"SolrConnector.NoFieldMappingSpecified") +
"</td></tr>\n"
);
}
+
+ String keepMetadataValue = "";
+ if(keepMetadata != null) {
+ keepMetadataValue =
keepMetadata.getAttributeValue(SolrConfig.ATTRIBUTE_VALUE);
+ keepMetadataValue = !keepMetadataValue.isEmpty() &&
keepMetadataValue != null && keepMetadataValue.equals("true") ? "checked" : "";
+ }
out.print(
" <tr class=\"formrow\"><td class=\"formseparator\"
colspan=\"3\"><hr/></td></tr>\n"+
" <tr class=\"formrow\">\n"+
@@ -2350,6 +2408,16 @@ public class SolrConnector extends org.a
" <nobr><input type=\"text\" size=\"15\"
name=\"solr_fieldmapping_target\" value=\"\"/></nobr>\n"+
" </td>\n"+
" </tr>\n"+
+ // Adding keep metadata
+ " <tr class=\"formrow\">\n"+
+ " <td class=\"formcolumncell\">\n"+
+ " <input type=\"checkbox\" "+keepMetadataValue+"
name=\"solr_keepallmetadata\" value=\"true\" />\n"+
+ " </td>\n"+
+ " <td class=\"formcolumncell\">\n"+
+
Messages.getAttributeString(locale,"SolrConnector.KeepAllMetadata")+
+ " </td>\n"+
+ " <td class=\"formcolumncell\">\n</td>\n"+
+ " </tr>\n"+
" </table>\n"+
" </td>\n"+
" </tr>\n"+
@@ -2376,6 +2444,9 @@ public class SolrConnector extends org.a
);
fieldCounter++;
}
+ else if(sn.getType().equals(SolrConfig.NODE_KEEPMETADATA)) {
+ out.print("<input type=\"hidden\" name=\"solr_keepallmetadata\"
value=\""+sn.getAttributeValue(SolrConfig.ATTRIBUTE_VALUE)+"\"/>\n");
+ }
}
out.print(
"<input type=\"hidden\" name=\"solr_fieldmapping_count\"
value=\""+Integer.toString(fieldCounter)+"\"/>\n"
@@ -2404,7 +2475,7 @@ public class SolrConnector extends org.a
while (i < os.getChildCount())
{
SpecificationNode node = os.getChild(i);
- if (node.getType().equals(SolrConfig.NODE_FIELDMAP))
+ if (node.getType().equals(SolrConfig.NODE_FIELDMAP) ||
node.getType().equals(SolrConfig.NODE_KEEPMETADATA))
os.removeChild(i);
else
i++;
@@ -2429,6 +2500,7 @@ public class SolrConnector extends org.a
}
i++;
}
+
String addop = variableContext.getParameter("solr_fieldmapping_op");
if (addop != null && addop.equals("Add"))
{
@@ -2441,6 +2513,18 @@ public class SolrConnector extends org.a
node.setAttribute(SolrConfig.ATTRIBUTE_TARGET,target);
os.addChild(os.getChildCount(),node);
}
+
+ // Gather the keep all metadata parameter to be the last one
+ SpecificationNode node = new
SpecificationNode(SolrConfig.NODE_KEEPMETADATA);
+ if(variableContext.getParameter("solr_keepallmetadata") != null &&
!variableContext.getParameter("solr_keepallmetadata").isEmpty()) {
+ node.setAttribute(SolrConfig.ATTRIBUTE_VALUE, "true");
+ }
+ else {
+ node.setAttribute(SolrConfig.ATTRIBUTE_VALUE, "false");
+ }
+ // Add the new keepallmetadata config parameter
+ os.addChild(os.getChildCount(), node);
+
}
return null;
}
Modified:
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties?rev=1556487&r1=1556486&r2=1556487&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties
(original)
+++
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties
Wed Jan 8 10:30:53 2014
@@ -14,6 +14,7 @@
# limitations under the License.
SolrConnector.Arguments=Arguments
+SolrConnector.KeepAllMetadata=Keep all metadata
SolrConnector.Arguments2=Arguments:
SolrConnector.Commits=Commits
SolrConnector.Documents=Documents
@@ -27,11 +28,10 @@ SolrConnector.SingleServer=Single server
SolrConnector.SolrCloud=SolrCloud
SolrConnector.ZookeeperHosts=ZooKeeper hosts:
SolrConnector.Host=Host
-SolrConnector.Port=Port
+SolrConnector.Port=Port\:
SolrConnector.ZnodePath=Znode path:
SolrConnector.Protocol=Protocol:
SolrConnector.ServerName=Server name:
-SolrConnector.Port=Port:
SolrConnector.WebApplicationName=Web application name:
SolrConnector.CoreName=Core/Collection name:
SolrConnector.CollectionName=Collection name:
@@ -92,16 +92,10 @@ SolrConnector.RemovePathMustStartWAChara
SolrConnector.StatusPathMustStartWACharacter=Status path must start with a
'/' character
SolrConnector.MaximumDocumentLengthMustBAnInteger=Maximum document length must
be an integer
SolrConnector.CommitWithinValueMustBeAnInteger=Commit-within value must be an
integer
-SolrConnector.PleaseSupplyAValidSolrServerName=Please supply a valid Solr
server name
-SolrConnector.SolrServerPortMustBeAValidInteger=Solr server port must be a
valid integer
-SolrConnector.WebApplicationNameCannotHaveCharacters=Web application name
cannot have '/' characters
SolrConnector.CoreNameCannotHaveCharacters=Core name cannot have '/' characters
-SolrConnector.WebApplicationMustBeSpecifiedIfCoreIsSpecified=Web application
must be specified if core is specified
-SolrConnector.UpdatePathMustStartWithACharacter=Update path must start with a
'/' character
SolrConnector.RemovePathMustStartWithACharacter=Remove path must start with a
'/' character
SolrConnector.StatusPathMustStartWithACharacter=Status path must start with a
'/' character
SolrConnector.MaximumDocumentLengthMustBeAnInteger=Maximum document length
must be an integer
-SolrConnector.CommitWithinValueMustBeAnInteger=Commit-within value must be an
integer
SolrConnector.ArgumentNameCannotBeAnEmptyString=Argument name cannot be an
empty string
SolrConnector.DeleteCert=Delete cert
SolrConnector.Delete=Delete
Added:
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/test/java/org/apache/manifoldcf/agents/output/solr/HttpPosterTest.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/test/java/org/apache/manifoldcf/agents/output/solr/HttpPosterTest.java?rev=1556487&view=auto
==============================================================================
---
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/test/java/org/apache/manifoldcf/agents/output/solr/HttpPosterTest.java
(added)
+++
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/test/java/org/apache/manifoldcf/agents/output/solr/HttpPosterTest.java
Wed Jan 8 10:30:53 2014
@@ -0,0 +1,238 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.manifoldcf.agents.output.solr;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import junit.framework.TestCase;
+import org.apache.log4j.Logger;
+import org.apache.manifoldcf.agents.interfaces.IOutputAddActivity;
+import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
+import org.apache.manifoldcf.agents.system.Logging;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.junit.runner.RunWith;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.when;
+import static org.powermock.api.mockito.PowerMockito.*;
+
+/**
+ * @Author: Alessandro Benedetti Date: 19/12/2013
+ */
+
+@RunWith( PowerMockRunner.class )
+@PrepareForTest( { HttpPoster.class } )
+public class HttpPosterTest
+ extends TestCase
+{
+
+
+ HttpPoster httpPosterToTest;
+
+ RepositoryDocument document;
+
+ IOutputAddActivity act;
+
+ @Override
+ public void setUp()
+ throws Exception
+ {
+
+ httpPosterToTest = spy(new HttpPoster( "zkHost", "collection1", 5000,
500, "update", "removePath", "statusPath",
+ "allowAttributeName", "denyAttributeName",
"idAttributeName",
+ "modifiedDateAttributeName",
"createdDateAttributeName", "indexedDateAttributeName",
+ "fileNameAttributeName",
"mimeTypeAttributeName", new Long( 5000 ), "true"));
+ }
+
+ /**
+ * Verify the IndexPost Method create the correct IngestThread class
+ * @throws Exception
+ */
+ public void testIndexPost()
+ throws Exception
+ {
+ String[] shareAcls = new String[]{ "shareAcl1", "shareAcl2" };
+ String[] shareDenyAcls = new String[]{ "denyShareAcl1",
"denyShareAcl2" };
+ String[] acls = new String[]{ "acl1", "acl2" };
+ String[] denyAcls = new String[]{ "denyAcl1", "denyAcl2" };
+
+ Logging.ingest = mock( Logger.class );
+ when( Logging.ingest.isDebugEnabled() ).thenReturn( false );
+ initRepositoryDocumentMock( shareAcls, shareDenyAcls, acls, denyAcls );
+
+ act = mock( IOutputAddActivity.class );
+ when(act.qualifyAccessToken( eq( "AuthorityString"), anyString()
)).thenAnswer(new Answer<String>() {
+ @Override
+ public String answer(InvocationOnMock invocation) throws Throwable
{
+ Object[] args = invocation.getArguments();
+ return (String) args[1];
+ }
+ });
+
+ Map<String, List<String>> sourceTargets = this.getMappingsMap();
+ Map<String, List<String>> streamParam = this.getArgumentsMap();
+
+ String commitWithin = "true";
+ HttpPoster.IngestThread
mockIngestionThread=mock(HttpPoster.IngestThread.class);
+ Mockito.doThrow(new RuntimeException()).when( mockIngestionThread
).run();
+
+ whenNew( HttpPoster.IngestThread.class).withArguments("Document Id",
document, streamParam, true, sourceTargets,shareAcls,
+ shareDenyAcls,
acls, denyAcls,commitWithin).thenReturn(mockIngestionThread);
+ httpPosterToTest.indexPost("Document Id", document, streamParam,
+ sourceTargets,true, "AuthorityString",act);
+
+ verifyNew(HttpPoster.IngestThread.class).withArguments("Document Id",
document, streamParam, true, sourceTargets, shareAcls,
+
shareDenyAcls, acls, denyAcls, commitWithin );
+ }
+
+ private void initRepositoryDocumentMock( String[] shareAcls, String[]
shareDenyAcls, String[] acls,
+ String[] denyAcls )
+ throws IOException
+ {
+ document = mock( RepositoryDocument.class );
+ List<String> fields = getFields();
+ Iterator<String> fieldsIterator = fields.iterator();
+ when( document.getFields() ).thenReturn( fieldsIterator );
+ when( document.getFieldAsStrings( "cm:description" ) ).thenReturn( new
String[]{ "description" } );
+ when( document.getFieldAsStrings( "cm:name" ) ).thenReturn( new
String[]{ "name" } );
+ when( document.getFieldAsStrings( "cm:title" ) ).thenReturn( new
String[]{ "title" } );
+ when( document.getFieldAsStrings( "extraMetadata1" ) ).thenReturn( new
String[]{ "value1" } );
+ when( document.getFieldAsStrings( "extraMetadata2" ) ).thenReturn( new
String[]{ "value2" } );
+ when( document.getFieldAsStrings( "extraMetadata3" ) ).thenReturn( new
String[]{ "value3" } );
+ when( document.getACL()).thenReturn(acls);
+ when( document.getShareACL()).thenReturn(shareAcls);
+ when( document.getShareDenyACL()).thenReturn( shareDenyAcls );
+ when( document.getDenyACL()).thenReturn(denyAcls);
+ }
+
+ /**
+ * inits the expected solr params for both the tests, in the first one we
expect the extra params not to be present
+ * because they are not in the mappings
+ *
+ * @param test2
+ * @return
+ * @throws java.io.UnsupportedEncodingException
+ */
+ private ModifiableSolrParams initExpectedSolrParams( Boolean test2 )
+ throws UnsupportedEncodingException
+ {
+ ModifiableSolrParams expectedParams = new ModifiableSolrParams();
+ expectedParams.add( "literal.idAttributeName", "document id" );
+ expectedParams.add( "literal.allowAttributeNameshare", "shareAcl1" );
+ expectedParams.add( "literal.allowAttributeNameshare", "shareAcl2" );
+ expectedParams.add( "literal.denyAttributeNameshare", "denyShareAcl1"
);
+ expectedParams.add( "literal.denyAttributeNameshare", "denyShareAcl2"
);
+ expectedParams.add( "literal.allowAttributeNamedocument", "acl1" );
+ expectedParams.add( "literal.allowAttributeNamedocument", "acl2" );
+ expectedParams.add( "literal.denyAttributeNamedocument", "denyAcl1" );
+ expectedParams.add( "literal.denyAttributeNamedocument", "denyAcl2" );
+ expectedParams.add( "stream.type", "text/plain" );
+ expectedParams.add( "literal.cm_description_s", "description" );
+ expectedParams.add( "literal.cm_title_s", "title" );
+ expectedParams.add( "literal.cm_name_s", "name" );
+ if ( test2 )
+ {
+ expectedParams.add( "literal.extraMetadata1", "value1" );
+ expectedParams.add( "literal.extraMetadata2", "value2" );
+ expectedParams.add( "literal.extraMetadata3", "value3" );
+ }
+ expectedParams.add( "commitWithin", "true" );
+ return expectedParams;
+ }
+
+ /**
+ * return a list of example metadata fields present in a mock document
+ *
+ * @return
+ */
+ private List<String> getFields()
+ {
+ List<String> fields = new ArrayList<String>();
+ fields.add( "cm:description" );
+ fields.add( "cm:title" );
+ fields.add( "cm:name" );
+ fields.add( "extraMetadata1" );
+ fields.add( "extraMetadata2" );
+ fields.add( "extraMetadata3" );
+ return fields;
+ }
+
+ /**
+ * returns a testing mapping map
+ *
+ * @return
+ */
+ private Map<String, List<String>> getMappingsMap()
+ {
+ Map<String, List<String>> sourceTargets = new HashMap<String,
List<String>>();
+ List<String> firstList = new ArrayList<String>();
+ firstList.add( "cm_description_s" );
+ List<String> secondList = new ArrayList<String>();
+ secondList.add( "cm_name_s" );
+ List<String> thirdList = new ArrayList<String>();
+ thirdList.add( "cm_title_s" );
+
+ sourceTargets.put( "cm:description", firstList );
+ sourceTargets.put( "cm:name", secondList );
+ sourceTargets.put( "cm:title", thirdList );
+ return sourceTargets;
+ }
+
+ /**
+ * returns a testing argument map
+ *
+ * @return
+ */
+ private Map<String, List<String>> getArgumentsMap()
+ {
+ Map<String, List<String>> sourceTargets = new HashMap<String,
List<String>>();
+ List<String> firstList = new ArrayList<String>();
+ firstList.add( "text/plain" );
+ sourceTargets.put( "stream.type", firstList );
+ return sourceTargets;
+ }
+}
Added:
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/test/java/org/apache/manifoldcf/agents/output/solr/IngestThreadTest.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/test/java/org/apache/manifoldcf/agents/output/solr/IngestThreadTest.java?rev=1556487&view=auto
==============================================================================
---
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/test/java/org/apache/manifoldcf/agents/output/solr/IngestThreadTest.java
(added)
+++
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/test/java/org/apache/manifoldcf/agents/output/solr/IngestThreadTest.java
Wed Jan 8 10:30:53 2014
@@ -0,0 +1,247 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.manifoldcf.agents.output.solr;
+
+import junit.framework.TestCase;
+import org.apache.log4j.Logger;
+import org.apache.manifoldcf.agents.interfaces.IOutputAddActivity;
+import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
+import org.apache.manifoldcf.agents.system.Logging;
+import org.apache.solr.client.solrj.SolrServer;
+import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
+import org.apache.solr.client.solrj.response.UpdateResponse;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.when;
+import static org.powermock.api.mockito.PowerMockito.doCallRealMethod;
+import static org.powermock.api.mockito.PowerMockito.mock;
+import static org.powermock.api.mockito.PowerMockito.spy;
+import static org.powermock.api.mockito.PowerMockito.*;
+
+/**
+ * @Author: Alessandro Benedetti Date: 19/12/2013
+ */
+@RunWith( PowerMockRunner.class )
+@PrepareForTest( { HttpPoster.IngestThread.class } )
+public class IngestThreadTest
+ extends TestCase
+{
+
+ HttpPoster.IngestThread ingestThreadToTest;
+
+ HttpPoster poster;
+
+ RepositoryDocument document;
+
+ IOutputAddActivity act;
+
+ @Override
+ public void setUp()
+ throws Exception
+ {
+
+ poster =new HttpPoster( "zkHost", "collection1", 5000, 500, "update",
"removePath", "statusPath",
+ "allowAttributeName", "denyAttributeName",
"idAttributeName",
+ "modifiedDateAttributeName",
"createdDateAttributeName", "indexedDateAttributeName",
+ "fileNameAttributeName",
"mimeTypeAttributeName", new Long( 5000 ), "true" );
+
+ Logging.ingest = mock( Logger.class );
+ when( Logging.ingest.isDebugEnabled() ).thenReturn( false );
+ document = mock( RepositoryDocument.class );
+ List<String> fields = getFields();
+ Iterator<String> fieldsIterator = fields.iterator();
+ when( document.getFields() ).thenReturn( fieldsIterator );
+ when( document.getFieldAsStrings( "cm:description" ) ).thenReturn( new
String[]{ "description" } );
+ when( document.getFieldAsStrings( "cm:name" ) ).thenReturn( new
String[]{ "name" } );
+ when( document.getFieldAsStrings( "cm:title" ) ).thenReturn( new
String[]{ "title" } );
+ when( document.getFieldAsStrings( "extraMetadata1" ) ).thenReturn( new
String[]{ "value1" } );
+ when( document.getFieldAsStrings( "extraMetadata2" ) ).thenReturn( new
String[]{ "value2" } );
+ when( document.getFieldAsStrings( "extraMetadata3" ) ).thenReturn( new
String[]{ "value3" } );
+
+ act = mock( IOutputAddActivity.class );
+
+ Map<String, List<String>> sourceTargets = this.getMappingsMap();
+ Map<String, List<String>> streamParam = this.getArgumentsMap();
+ String[] shareAcls = new String[]{ "shareAcl1", "shareAcl2" };
+ String[] shareDenyAcls = new String[]{ "denyShareAcl1",
"denyShareAcl2" };
+ String[] acls = new String[]{ "acl1", "acl2" };
+ String[] denyAcls = new String[]{ "denyAcl1", "denyAcl2" };
+ String commitWithin = "true";
+
+ ingestThreadToTest = spy(
+ poster.new IngestThread( "document id", document, streamParam,
true, sourceTargets, shareAcls,
+ shareDenyAcls, acls, denyAcls,
commitWithin ) );
+
+ }
+
+
+ public void testIndexPostNotKeepMetadata()
+ throws Exception
+ {
+ this.indexPostNotKeepMetadata( true );
+ this.indexPostNotKeepMetadata( false );
+
+ }
+
+ /**
+ * Verify the behaviour, when the keep variable is set to true, we want to
send to Solr all the metadata fields
+ *
+ * @param keep
+ * @throws Exception
+ */
+ private void indexPostNotKeepMetadata( Boolean keep )
+ throws Exception
+ {
+ ingestThreadToTest.setKeepAllMetadata( keep );
+
+ ContentStreamUpdateRequest contentStreamUpdateRequest = mock(
ContentStreamUpdateRequest.class );
+ whenNew( ContentStreamUpdateRequest.class ).withArguments( anyString()
).thenReturn(
+ contentStreamUpdateRequest );
+ UpdateResponse mockResponse = mock( UpdateResponse.class );
+ when( contentStreamUpdateRequest.process( any( SolrServer.class ) )
).thenReturn( mockResponse );
+ doCallRealMethod().when( contentStreamUpdateRequest ).setParams( any(
ModifiableSolrParams.class ) );
+ doCallRealMethod().when( contentStreamUpdateRequest ).getParams();
+ ingestThreadToTest.run();
+ verifyNew( ContentStreamUpdateRequest.class, atLeastOnce()
).withArguments( anyString() );
+
+ ModifiableSolrParams expectedParams = initExpectedSolrParams( keep );
+ assertEqualsModifiableSolrParams( expectedParams,
contentStreamUpdateRequest.getParams() );
+ verify( contentStreamUpdateRequest ).process( any( SolrServer.class )
);
+
+ }
+
+ /**
+ * asserts that 2 ModifiableSolrParams are equals
+ *
+ * @param expected
+ * @param actual
+ * @return
+ */
+ private boolean assertEqualsModifiableSolrParams( ModifiableSolrParams
expected, ModifiableSolrParams actual )
+ {
+ Set<String> expectedParameterNames = expected.getParameterNames();
+ Set<String> actualParameterNames = actual.getParameterNames();
+ int expectedSize = expectedParameterNames.size();
+ assertEquals( expectedSize, actualParameterNames.size() );
+ for ( String parameterName : expectedParameterNames )
+ {
+ assertEquals( expected.get( parameterName ), actual.get(
parameterName ) );
+ }
+ return true;
+ }
+
+ /**
+ * inits the expected solr params for both the tests, in the first one we
expect the extra params not to be present
+ * because they are not in the mappings
+ *
+ * @param test2
+ * @return
+ * @throws UnsupportedEncodingException
+ */
+ private ModifiableSolrParams initExpectedSolrParams( Boolean test2 )
+ throws UnsupportedEncodingException
+ {
+ ModifiableSolrParams expectedParams = new ModifiableSolrParams();
+ expectedParams.add( "literal.idAttributeName", "document id" );
+ expectedParams.add( "literal.allowAttributeNameshare", "shareAcl1" );
+ expectedParams.add( "literal.allowAttributeNameshare", "shareAcl2" );
+ expectedParams.add( "literal.denyAttributeNameshare", "denyShareAcl1"
);
+ expectedParams.add( "literal.denyAttributeNameshare", "denyShareAcl2"
);
+ expectedParams.add( "literal.allowAttributeNamedocument", "acl1" );
+ expectedParams.add( "literal.allowAttributeNamedocument", "acl2" );
+ expectedParams.add( "literal.denyAttributeNamedocument", "denyAcl1" );
+ expectedParams.add( "literal.denyAttributeNamedocument", "denyAcl2" );
+ expectedParams.add( "stream.type", "text/plain" );
+ expectedParams.add( "literal.cm_description_s", "description" );
+ expectedParams.add( "literal.cm_title_s", "title" );
+ expectedParams.add( "literal.cm_name_s", "name" );
+ if ( test2 )
+ {
+ expectedParams.add( "literal.extraMetadata1", "value1" );
+ expectedParams.add( "literal.extraMetadata2", "value2" );
+ expectedParams.add( "literal.extraMetadata3", "value3" );
+ }
+ expectedParams.add( "commitWithin", "true" );
+ return expectedParams;
+ }
+
+ /**
+ * return a list of example metadata fields present in a mock document
+ *
+ * @return
+ */
+ private List<String> getFields()
+ {
+ List<String> fields = new ArrayList<String>();
+ fields.add( "cm:description" );
+ fields.add( "cm:title" );
+ fields.add( "cm:name" );
+ fields.add( "extraMetadata1" );
+ fields.add( "extraMetadata2" );
+ fields.add( "extraMetadata3" );
+ return fields;
+ }
+
+ /**
+ * returns a testing mapping map
+ *
+ * @return
+ */
+ private Map<String, List<String>> getMappingsMap()
+ {
+ Map<String, List<String>> sourceTargets = new HashMap<String,
List<String>>();
+ List<String> firstList = new ArrayList<String>();
+ firstList.add( "cm_description_s" );
+ List<String> secondList = new ArrayList<String>();
+ secondList.add( "cm_name_s" );
+ List<String> thirdList = new ArrayList<String>();
+ thirdList.add( "cm_title_s" );
+
+ sourceTargets.put( "cm:description", firstList );
+ sourceTargets.put( "cm:name", secondList );
+ sourceTargets.put( "cm:title", thirdList );
+ return sourceTargets;
+ }
+
+ /**
+ * returns a testing argument map
+ *
+ * @return
+ */
+ private Map<String, List<String>> getArgumentsMap()
+ {
+ Map<String, List<String>> sourceTargets = new HashMap<String,
List<String>>();
+ List<String> firstList = new ArrayList<String>();
+ firstList.add( "text/plain" );
+ sourceTargets.put( "stream.type", firstList );
+ return sourceTargets;
+ }
+}
Added:
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/test/java/org/apache/manifoldcf/agents/output/solr/SolrConnectorTest.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/test/java/org/apache/manifoldcf/agents/output/solr/SolrConnectorTest.java?rev=1556487&view=auto
==============================================================================
---
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/test/java/org/apache/manifoldcf/agents/output/solr/SolrConnectorTest.java
(added)
+++
manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/test/java/org/apache/manifoldcf/agents/output/solr/SolrConnectorTest.java
Wed Jan 8 10:30:53 2014
@@ -0,0 +1,87 @@
+
+package org.apache.manifoldcf.agents.output.solr;
+
+import junit.framework.TestCase;
+import org.apache.manifoldcf.agents.interfaces.IOutputAddActivity;
+import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
+import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
+import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.mockito.Matchers.*;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.powermock.api.mockito.PowerMockito.mock;
+import static org.powermock.api.mockito.PowerMockito.spy;
+
+/**
+ * @Author: Alessandro Benedetti
+ * Date: 18/12/2013
+ */
+public class SolrConnectorTest extends TestCase{
+
+ SolrConnector connectorToTest;
+ HttpPoster poster;
+ RepositoryDocument document;
+ IOutputAddActivity act;
+
+ @Override
+ public void setUp() throws Exception {
+ connectorToTest=spy(new SolrConnector());
+ poster=mock(HttpPoster.class);
+ document=mock(RepositoryDocument.class);
+ act=mock(IOutputAddActivity.class);
+
when(poster.indexPost(anyString(),eq(document),anyMap(),anyMap(),anyBoolean(),anyString(),eq(act))).thenReturn(true);
+
+ }
+
+ /**
+ * Test the AddOrReplaceDocument with an example test string in input
+ * @throws ManifoldCFException
+ * @throws ServiceInterruption
+ */
+ public void testAddOrReplaceDocument() throws ManifoldCFException,
ServiceInterruption {
+ Map<String, List<String>> expectedSourceTargets = getMappingsMap();
+ Map<String,List<String>> expectedStreamParams = getStreamTypeMap();
+
+ connectorToTest.poster=poster;
+ String outputDescription =
"1+stream.type=text/plain=+3+cm:description=cm_description_s=+cm:name=cm_name_s=+cm:title=cm_title_s=+1+keepAllMetadata=true=+";
+ connectorToTest.addOrReplaceDocument("Document Id",
outputDescription,document,"",act);
+ verify(poster).indexPost(eq("Document
Id"),eq(document),eq(expectedStreamParams),eq(expectedSourceTargets),eq(true),eq(""),eq(act));
+ }
+
+ /**
+ * returns the expected mappings map for the input string in test
+ * @return
+ */
+ private Map<String, List<String>> getMappingsMap() {
+ Map<String, List<String>> sourceTargets = new HashMap<String,
List<String>>();
+ List<String> firstList=new ArrayList<String>();
+ firstList.add("cm_description_s");
+ List<String> secondList=new ArrayList<String>();
+ secondList.add("cm_name_s");
+ List<String> thirdList=new ArrayList<String>();
+ thirdList.add("cm_title_s");
+
+ sourceTargets.put("cm:description",firstList);
+ sourceTargets.put("cm:name",secondList);
+ sourceTargets.put("cm:title",thirdList);
+ return sourceTargets;
+ }
+
+ /**
+ * returns the expected mappings map for the input string in test
+ * @return
+ */
+ private Map<String, List<String>> getStreamTypeMap() {
+ Map<String, List<String>> sourceTargets = new HashMap<String,
List<String>>();
+ List<String> firstList=new ArrayList<String>();
+ firstList.add("text/plain");
+ sourceTargets.put("stream.type",firstList);
+ return sourceTargets;
+ }
+}
Modified: manifoldcf/branches/CONNECTORS-840/connectors/solr/pom.xml
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-840/connectors/solr/pom.xml?rev=1556487&r1=1556486&r2=1556487&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-840/connectors/solr/pom.xml (original)
+++ manifoldcf/branches/CONNECTORS-840/connectors/solr/pom.xml Wed Jan 8
10:30:53 2014
@@ -101,5 +101,25 @@
<artifactId>httpclient</artifactId>
<version>${httpcomponent.httpclient.version}</version>
</dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${junit.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>${powerMock.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito</artifactId>
+ <version>${powerMock.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
Modified: manifoldcf/branches/CONNECTORS-840/pom.xml
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-840/pom.xml?rev=1556487&r1=1556486&r2=1556487&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-840/pom.xml (original)
+++ manifoldcf/branches/CONNECTORS-840/pom.xml Wed Jan 8 10:30:53 2014
@@ -38,6 +38,7 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<!-- dependency versions -->
<junit.version>4.8.2</junit.version>
+ <powerMock.version>1.5.2</powerMock.version>
<postgresql.version>9.1-901.jdbc4</postgresql.version>
<mysql.version>5.1.18</mysql.version>
<hsqldb.version>2.3.1</hsqldb.version>
