Author: kwright
Date: Fri Sep 19 10:05:13 2014
New Revision: 1626166
URL: http://svn.apache.org/r1626166
Log:
Pull up changes for CONNECTORS-956 from trunk. Also add switch on Schema tab
to allow selection of old vs new behavior.
Modified:
manifoldcf/branches/dev_1x/ (props changed)
manifoldcf/branches/dev_1x/CHANGES.txt
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedHttpSolrServer.java
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_zh_CN.properties
Propchange: manifoldcf/branches/dev_1x/
------------------------------------------------------------------------------
Merged /manifoldcf/trunk:r1625910
Modified: manifoldcf/branches/dev_1x/CHANGES.txt
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/CHANGES.txt?rev=1626166&r1=1626165&r2=1626166&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/CHANGES.txt (original)
+++ manifoldcf/branches/dev_1x/CHANGES.txt Fri Sep 19 10:05:13 2014
@@ -18,6 +18,9 @@ for document filtering.
CONNECTORS-1038: Handle multiple pool closes properly.
(Karl Wright)
+CONNECTORS-956: Change how Solr field name escaping is done.
+(Piergiorgio Lucidi, Shinichiro Abe, Edgardo Ambrosi, Karl Wright)
+
CONNECTORS-1036: Zookeeper service handling also has
ephemeral nodes which need to be tied to sessions.
(Karl Wright)
Modified:
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java?rev=1626166&r1=1626165&r2=1626166&view=diff
==============================================================================
---
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
(original)
+++
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
Fri Sep 19 10:05:13 2014
@@ -102,6 +102,9 @@ public class HttpPoster
// Whether we use extract/update handler or not
private final boolean useExtractUpdateHandler;
+ // Use url encoding for field names
+ private final boolean useUrlEncoding;
+
// Document max length
private final Long maxDocumentLength;
@@ -126,7 +129,8 @@ public class HttpPoster
String modifiedDateAttributeName, String createdDateAttributeName, String
indexedDateAttributeName,
String fileNameAttributeName, String mimeTypeAttributeName, String
contentAttributeName,
Long maxDocumentLength,
- String commitWithin, boolean useExtractUpdateHandler)
+ String commitWithin, boolean useExtractUpdateHandler,
+ boolean useUrlEncoding)
throws ManifoldCFException
{
// These are the paths to the handlers in Solr that deal with the actions
we need to do
@@ -146,7 +150,7 @@ public class HttpPoster
this.mimeTypeAttributeName = mimeTypeAttributeName;
this.contentAttributeName = contentAttributeName;
this.useExtractUpdateHandler = useExtractUpdateHandler;
-
+ this.useUrlEncoding = useUrlEncoding;
this.maxDocumentLength = maxDocumentLength;
try
@@ -174,7 +178,8 @@ public class HttpPoster
String modifiedDateAttributeName, String createdDateAttributeName, String
indexedDateAttributeName,
String fileNameAttributeName, String mimeTypeAttributeName, String
contentAttributeName,
IKeystoreManager keystoreManager, Long maxDocumentLength,
- String commitWithin, boolean useExtractUpdateHandler)
+ String commitWithin, boolean useExtractUpdateHandler,
+ boolean useUrlEncoding)
throws ManifoldCFException
{
// These are the paths to the handlers in Solr that deal with the actions
we need to do
@@ -194,6 +199,7 @@ public class HttpPoster
this.mimeTypeAttributeName = mimeTypeAttributeName;
this.contentAttributeName = contentAttributeName;
this.useExtractUpdateHandler = useExtractUpdateHandler;
+ this.useUrlEncoding = useUrlEncoding;
this.maxDocumentLength = maxDocumentLength;
@@ -763,26 +769,29 @@ public class HttpPoster
return new String[0];
}
- /** Preprocess field name.
- * SolrJ has a bug where it does not URL-escape field names. This causes
carnage for
- * ManifoldCF, because it results in IllegalArgumentExceptions getting thrown
deep in SolrJ.
- * See CONNECTORS-630.
- * In order to get around this, we need to URL-encode argument names, at
least until the underlying
- * SolrJ issue is fixed.
- */
- protected static String preEncode(String fieldName)
+ /** Conditionally encode */
+ protected String conditionallyEncode(String fieldName)
{
- return URLEncoder.encode(fieldName);
+ if (!useUrlEncoding)
+ return fieldName;
+ try
+ {
+ return java.net.URLEncoder.encode(fieldName, "UTF-8");
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException("Could not find utf-8 encoding!");
+ }
}
/** Write a field */
- protected static void writeField(ModifiableSolrParams out, String fieldName,
String[] fieldValues)
+ protected void writeField(ModifiableSolrParams out, String fieldName,
String[] fieldValues)
{
- out.add(preEncode(fieldName), fieldValues);
+ out.add(conditionallyEncode(fieldName), fieldValues);
}
/** Write a field */
- protected static void writeField(ModifiableSolrParams out, String fieldName,
List<String> fieldValues)
+ protected void writeField(ModifiableSolrParams out, String fieldName,
List<String> fieldValues)
{
String[] values = new String[fieldValues.size()];
int i = 0;
@@ -793,9 +802,9 @@ public class HttpPoster
}
/** Write a field */
- protected static void writeField(ModifiableSolrParams out, String fieldName,
String fieldValue)
+ protected void writeField(ModifiableSolrParams out, String fieldName, String
fieldValue)
{
- out.add(preEncode(fieldName), fieldValue);
+ out.add(conditionallyEncode(fieldName), fieldValue);
}
/** Output an acl level */
@@ -1154,7 +1163,7 @@ public class HttpPoster
Iterator<String> iter = document.getFields();
while (iter.hasNext())
{
- String fieldName = iter.next();
+ String fieldName = makeSafeLuceneField(iter.next());
List<String> mappings = sourceTargets.get(fieldName);
if (mappings != null)
for (String newFieldName : mappings)
@@ -1182,7 +1191,7 @@ public class HttpPoster
Iterator<String> iter = document.getFields();
while (iter.hasNext())
{
- String fieldName = iter.next();
+ String fieldName = makeSafeLuceneField(iter.next());
List<String> mappings = sourceTargets.get(fieldName);
if (mappings != null)
for (String newFieldName : mappings)
@@ -1611,5 +1620,38 @@ public class HttpPoster
}
}
+ /** See CONNECTORS-956. Make a safe lucene field name from a possibly
+ * unsafe input field name from a repository connector.
+ */
+ protected String makeSafeLuceneField(String inputField)
+ {
+ if (useUrlEncoding)
+ return inputField;
+ StringBuilder sb = new StringBuilder();
+ boolean isFirst = true;
+ for (int i = 0; i < inputField.length(); i++)
+ {
+ char x = inputField.charAt(i);
+ if (isFirst && !Character.isJavaIdentifierStart(x) || !isFirst &&
!Character.isJavaIdentifierPart(x))
+ {
+ // Check for exceptions for Lucene
+ if (!isFirst && (x == '.' || x == '-'))
+ sb.append(x);
+ else
+ sb.append('_');
+ }
+ else
+ {
+ // Check for exceptions for Lucene
+ if (isFirst && x == '$')
+ sb.append('_');
+ else
+ sb.append(x);
+ }
+ isFirst = false;
+ }
+ return sb.toString();
+ }
+
}
Modified:
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedHttpSolrServer.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedHttpSolrServer.java?rev=1626166&r1=1626165&r2=1626166&view=diff
==============================================================================
---
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedHttpSolrServer.java
(original)
+++
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedHttpSolrServer.java
Fri Sep 19 10:05:13 2014
@@ -26,6 +26,7 @@ import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
+import java.net.URLEncoder;
import org.apache.http.Header;
import org.apache.http.HttpResponse;
@@ -140,7 +141,7 @@ public class ModifiedHttpSolrServer exte
if( streams != null ) {
throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,
"GET can't send streams!" );
}
- method = new HttpGet( baseUrl + path + ClientUtils.toQueryString(
params, false ) );
+ method = new HttpGet( baseUrl + path + toQueryString( params,
false ) );
}
else if( SolrRequest.METHOD.POST == request.getMethod() ) {
@@ -211,7 +212,7 @@ public class ModifiedHttpSolrServer exte
}
// It is has one stream, it is the post body, put the params in
the URL
else {
- String pstr = ClientUtils.toQueryString(params, false);
+ String pstr = toQueryString(params, false);
HttpPost post = new HttpPost(url + pstr);
// Single stream as body
@@ -364,4 +365,35 @@ public class ModifiedHttpSolrServer exte
this.followRedirects = followRedirects;
}
+ public static String toQueryString( SolrParams params, boolean xml ) {
+ StringBuilder sb = new StringBuilder(128);
+ try {
+ String amp = xml ? "&" : "&";
+ boolean first=true;
+ Iterator<String> names = params.getParameterNamesIterator();
+ while( names.hasNext() ) {
+ String key = names.next();
+ String[] valarr = params.getParams( key );
+ if( valarr == null ) {
+ sb.append( first?"?":amp );
+ sb.append( URLEncoder.encode(key, "UTF-8") );
+ first=false;
+ }
+ else {
+ for (String val : valarr) {
+ sb.append( first? "?":amp );
+ sb.append(key);
+ if( val != null ) {
+ sb.append('=');
+ sb.append( URLEncoder.encode( val, "UTF-8" ) );
+ }
+ first=false;
+ }
+ }
+ }
+ }
+ catch (IOException e) {throw new RuntimeException(e);} // can't happen
+ return sb.toString();
+ }
+
}
Modified:
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java?rev=1626166&r1=1626165&r2=1626166&view=diff
==============================================================================
---
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
(original)
+++
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
Fri Sep 19 10:05:13 2014
@@ -113,6 +113,8 @@ public class SolrConfig
public static final String PARAM_EXCLUDEDMIMETYPES="Excluded mime types";
/** Parameter describing the use of Extract Update handler */
public static final String PARAM_EXTRACTUPDATE = "Use extract update
handler";
+ /** Parameter describing whether field names should be url encoded */
+ public static final String PARAM_URLENCODING = "Use url encoding for field
names";
/** Optional content field (if not using extract update handler) */
public static final String PARAM_CONTENTFIELD = "Solr content field name";
/** Node describing an argument */
Modified:
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java?rev=1626166&r1=1626165&r2=1626166&view=diff
==============================================================================
---
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
(original)
+++
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
Fri Sep 19 10:05:13 2014
@@ -93,6 +93,8 @@ public class SolrConnector extends org.a
protected String contentAttributeName = null;
/** Use extractiing update handler? */
protected boolean useExtractUpdateHandler = true;
+ /** Use URL encoding for field names? */
+ protected boolean useUrlEncoding = true;
/** Whether or not to commit */
protected boolean doCommits = false;
@@ -185,6 +187,7 @@ public class SolrConnector extends org.a
mimeTypeAttributeName = null;
contentAttributeName = null;
useExtractUpdateHandler = true;
+ useUrlEncoding = true;
super.disconnect();
}
@@ -242,6 +245,12 @@ public class SolrConnector extends org.a
if (contentAttributeName == null && !useExtractUpdateHandler)
throw new ManifoldCFException("Content attribute name required for
non-extract-update indexing");
+ String useUrlEncodingValue =
params.getParameter(SolrConfig.PARAM_URLENCODING);
+ if (useUrlEncodingValue == null || useUrlEncodingValue.length() == 0)
+ useUrlEncoding = true;
+ else
+ useUrlEncoding = !useUrlEncodingValue.equals("false");
+
String commits = params.getParameter(SolrConfig.PARAM_COMMITS);
if (commits == null || commits.length() == 0)
commits = "true";
@@ -351,7 +360,8 @@ public class SolrConnector extends org.a
allowAttributeName,denyAttributeName,idAttributeName,
modifiedDateAttributeName,createdDateAttributeName,indexedDateAttributeName,
fileNameAttributeName,mimeTypeAttributeName,contentAttributeName,
-
keystoreManager,maxDocumentLength,commitWithin,useExtractUpdateHandler);
+
keystoreManager,maxDocumentLength,commitWithin,useExtractUpdateHandler,
+ useUrlEncoding);
}
catch (NumberFormatException e)
@@ -406,7 +416,8 @@ public class SolrConnector extends org.a
allowAttributeName,denyAttributeName,idAttributeName,
modifiedDateAttributeName,createdDateAttributeName,indexedDateAttributeName,
fileNameAttributeName,mimeTypeAttributeName,contentAttributeName,
- maxDocumentLength,commitWithin,useExtractUpdateHandler);
+ maxDocumentLength,commitWithin,useExtractUpdateHandler,
+ useUrlEncoding);
}
catch (NumberFormatException e)
@@ -1031,6 +1042,10 @@ public class SolrConnector extends org.a
if (useExtractUpdate == null || useExtractUpdate.length() == 0)
useExtractUpdate = "true";
+ String useUrlEncoding =
parameters.getParameter(SolrConfig.PARAM_URLENCODING);
+ if (useUrlEncoding == null || useUrlEncoding.length() == 0)
+ useUrlEncoding = "true";
+
String realm = parameters.getParameter(SolrConfig.PARAM_REALM);
if (realm == null)
realm = "";
@@ -1512,6 +1527,28 @@ public class SolrConnector extends org.a
" <input name=\"contentfield\" type=\"text\" size=\"32\"
value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(contentField)+"\"/>\n"+
" </td>\n"+
" </tr>\n"+
+" <tr><td colspan=\"2\" class=\"separator\"><hr/></td></tr>\n"+
+" <tr>\n"+
+" <td class=\"description\"><nobr>" +
Messages.getBodyString(locale,"SolrConnector.UseURLEncodingForFieldNames") +
"</nobr></td>\n"+
+" <td class=\"value\">\n"+
+" <input name=\"useurlencodingcheckbox\" type=\"hidden\"
value=\"true\"/>\n"+
+" <input name=\"useurlencodingpresent\" type=\"hidden\"
value=\"true\"/>\n"
+ );
+ if (!useUrlEncoding.equals("false"))
+ {
+ out.print(
+" <input name=\"useurlencoding\" type=\"checkbox\" value=\"true\"
checked=\"true\"/>\n"
+ );
+ }
+ else
+ {
+ out.print(
+" <input name=\"useurlencoding\" type=\"checkbox\" value=\"true\"/>\n"
+ );
+ }
+ out.print(
+" </td>\n"+
+" </tr>\n"+
"</table>\n"
);
}
@@ -1527,7 +1564,9 @@ public class SolrConnector extends org.a
"<input type=\"hidden\" name=\"contentfield\"
value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(contentField)+"\"/>\n"+
"<input name=\"extractupdatecheckbox\" type=\"hidden\" value=\"false\"/>\n"+
"<input type=\"hidden\" name=\"extractupdate\"
value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(useExtractUpdate)+"\"/>\n"+
-"<input name=\"extractupdatepresent\" type=\"hidden\" value=\"true\"/>\n"
+"<input name=\"extractupdatepresent\" type=\"hidden\" value=\"true\"/>\n"+
+"<input type=\"hidden\" name=\"useurlencoding\"
value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(useUrlEncoding)+"\"/>\n"+
+"<input name=\"useurlencodingpresent\" type=\"hidden\" value=\"true\"/>\n"
);
}
@@ -1840,6 +1879,15 @@ public class SolrConnector extends org.a
parameters.setParameter(SolrConfig.PARAM_EXTRACTUPDATE,extractUpdate);
}
+ String useUrlEncodingPresent =
variableContext.getParameter("useurlencodingpresent");
+ if (useUrlEncodingPresent != null)
+ {
+ String useUrlEncoding = variableContext.getParameter("useurlencoding");
+ if (useUrlEncoding == null || useUrlEncoding.length() == 0)
+ useUrlEncoding = "false";
+ parameters.setParameter(SolrConfig.PARAM_URLENCODING,useUrlEncoding);
+ }
+
String realm = variableContext.getParameter("realm");
if (realm != null)
parameters.setParameter(SolrConfig.PARAM_REALM,realm);
@@ -2813,6 +2861,11 @@ public class SolrConnector extends org.a
else
sb.append('-');
+ if (useUrlEncoding)
+ sb.append('+');
+ else
+ sb.append('-');
+
// Length limitation. We pack this because when it is changed we want
to be sure we get any previously excluded documents.
if (maxDocumentLength != null)
{
Modified:
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties?rev=1626166&r1=1626165&r2=1626166&view=diff
==============================================================================
---
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties
(original)
+++
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties
Fri Sep 19 10:05:13 2014
@@ -104,4 +104,4 @@ SolrConnector.DeleteArgument=Delete argu
SolrConnector.DeleteZookeeperHost=Delete ZooKeeper host #
SolrConnector.FieldMapMustHaveNonNullSource=Field map must have non-null source
SolrConnector.DeleteFieldMapping=Delete field mapping #
-
+SolrConnector.UseURLEncodingForFieldNames=Use URL encoding for field names:
Modified:
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties?rev=1626166&r1=1626165&r2=1626166&view=diff
==============================================================================
---
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties
(original)
+++
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties
Fri Sep 19 10:05:13 2014
@@ -104,4 +104,5 @@ SolrConnector.DeleteArgument=å¼æ�
SolrConnector.DeleteZookeeperHost=ZooKeeperãã¹ããåé¤ #
SolrConnector.FieldMapMustHaveNonNullSource=ãã£ã¼ã«ãããããå
¥åãã¦ãã ãã
SolrConnector.DeleteFieldMapping=ãã£ã¼ã«ãããããåé¤ #
+SolrConnector.UseURLEncodingForFieldNames=Use URL encoding for field names:
Modified:
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_zh_CN.properties
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_zh_CN.properties?rev=1626166&r1=1626165&r2=1626166&view=diff
==============================================================================
---
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_zh_CN.properties
(original)
+++
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_zh_CN.properties
Fri Sep 19 10:05:13 2014
@@ -104,4 +104,5 @@ SolrConnector.DeleteArgument=å é�
SolrConnector.DeleteZookeeperHost=å é¤ZooKeeperä¸»æº #
SolrConnector.FieldMapMustHaveNonNullSource=请è¾å
¥å段æ å°
SolrConnector.DeleteFieldMapping=å é¤å段æ å° #
+SolrConnector.UseURLEncodingForFieldNames=Use URL encoding for field names:
