Author: kwright
Date: Fri Aug 15 10:23:12 2014
New Revision: 1618144
URL: http://svn.apache.org/r1618144
Log:
Fix for CONNECTORS-1010.
Modified:
manifoldcf/trunk/CHANGES.txt
manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
Modified: manifoldcf/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1618144&r1=1618143&r2=1618144&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Fri Aug 15 10:23:12 2014
@@ -5,6 +5,10 @@ $Id$
======================= Release 1.7 =====================
+CONNECTORS-1010: Use uri hash instead of full URI as lock key
+to prevent "file name too long" errors on windows.
+(Erlend Garåsen, Shigeki Kobayashi, Karl Wright)
+
CONNECTORS-1007: Fix CMIS connector so that tests pass.
(Karl Wright)
Modified:
manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java?rev=1618144&r1=1618143&r2=1618144&view=diff
==============================================================================
---
manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
(original)
+++
manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
Fri Aug 15 10:23:12 2014
@@ -1036,13 +1036,15 @@ public class IncrementalIngester extends
}
String[] lockArray = new String[validURIcount];
String[] validURIArray = new String[validURIcount];
+ String[] validURIHashArray = new String[validURIcount];
validURIcount = 0;
for (int i = 0; i < uris.length; i++)
{
if (uris[i] != null && uris[i].getURI() != null)
{
validURIArray[validURIcount] = uris[i].getURI();
- lockArray[validURIcount] =
outputConnectionName+":"+validURIArray[validURIcount];
+ validURIHashArray[validURIcount] = uris[i].getURIHash();
+ lockArray[validURIcount] =
outputConnectionName+":"+validURIHashArray[validURIcount];
validURIcount++;
}
}
@@ -1079,9 +1081,11 @@ public class IncrementalIngester extends
Set<String> docURIValues = new HashSet<String>();
for (String docDBString : validURIArray)
{
- String docDBHashString = ManifoldCF.hash(docDBString);
docURIValues.add(docDBString);
- docURIHashValues.add(docDBHashString);
+ }
+ for (String docDBString : validURIHashArray)
+ {
+ docURIHashValues.add(docDBString);
}
// Now, perform n queries, each of them no larger the maxInClause in
length.
@@ -1248,13 +1252,15 @@ public class IncrementalIngester extends
}
String[] lockArray = new String[validURIcount];
String[] validURIArray = new String[validURIcount];
+ String[] validURIHashArray = new String[validURIcount];
validURIcount = 0;
for (int i = 0; i < uris.length; i++)
{
if (uris[i] != null && uris[i].getURI() != null)
{
validURIArray[validURIcount] = uris[i].getURI();
- lockArray[validURIcount] =
outputConnectionName+":"+validURIArray[validURIcount];
+ validURIHashArray[validURIcount] = uris[i].getURIHash();
+ lockArray[validURIcount] =
outputConnectionName+":"+validURIHashArray[validURIcount];
validURIcount++;
}
}
@@ -1291,9 +1297,11 @@ public class IncrementalIngester extends
Set<String> docURIValues = new HashSet<String>();
for (String docDBString : validURIArray)
{
- String docDBHashString = ManifoldCF.hash(docDBString);
docURIValues.add(docDBString);
- docURIHashValues.add(docDBHashString);
+ }
+ for (String docDBString : validURIHashArray)
+ {
+ docURIHashValues.add(docDBString);
}
// Now, perform n queries, each of them no larger the maxInClause in
length.
@@ -2237,7 +2245,7 @@ public class IncrementalIngester extends
new MultiClause(docKeyField,list),
new UnitaryClause(outputConnNameField,outputConnectionName)});
- IResultSet set = performQuery("SELECT
"+docKeyField+","+docURIField+","+lastOutputVersionField+" FROM
"+getTableName()+" WHERE "+
+ IResultSet set = performQuery("SELECT
"+docKeyField+","+docURIField+","+uriHashField+","+lastOutputVersionField+"
FROM "+getTableName()+" WHERE "+
query,newList,null,null);
// Go through list and put into buckets.
@@ -2245,14 +2253,17 @@ public class IncrementalIngester extends
{
IResultRow row = set.getRow(i);
String docHash = row.getValue(docKeyField).toString();
- Integer position = (Integer)map.get(docHash);
+ Integer position = map.get(docHash);
if (position != null)
{
String lastURI = (String)row.getValue(docURIField);
if (lastURI != null && lastURI.length() == 0)
lastURI = null;
+ String lastURIHash = (String)row.getValue(uriHashField);
+ if (lastURIHash != null && lastURIHash.length() == 0)
+ lastURIHash = null;
String lastOutputVersion =
(String)row.getValue(lastOutputVersionField);
- rval[position.intValue()] = new DeleteInfo(lastURI,lastOutputVersion);
+ rval[position.intValue()] = new
DeleteInfo(lastURI,lastURIHash,lastOutputVersion);
}
}
}
@@ -2283,7 +2294,7 @@ public class IncrementalIngester extends
new UnitaryClause(outputConnNameField,outputConnectionName),
((componentHash==null)?new NullCheckClause(componentHashField,true):new
UnitaryClause(componentHashField,componentHash))});
- IResultSet set = performQuery("SELECT
"+docKeyField+","+docURIField+","+lastOutputVersionField+" FROM
"+getTableName()+" WHERE "+
+ IResultSet set = performQuery("SELECT
"+docKeyField+","+docURIField+","+uriHashField+","+lastOutputVersionField+"
FROM "+getTableName()+" WHERE "+
query,newList,null,null);
// Go through list and put into buckets.
@@ -2297,8 +2308,11 @@ public class IncrementalIngester extends
String lastURI = (String)row.getValue(docURIField);
if (lastURI != null && lastURI.length() == 0)
lastURI = null;
+ String lastURIHash = (String)row.getValue(uriHashField);
+ if (lastURIHash != null && lastURIHash.length() == 0)
+ lastURIHash = null;
String lastOutputVersion =
(String)row.getValue(lastOutputVersionField);
- rval[position.intValue()] = new DeleteInfo(lastURI,lastOutputVersion);
+ rval[position.intValue()] = new
DeleteInfo(lastURI,lastURIHash,lastOutputVersion);
}
}
}
@@ -2351,11 +2365,13 @@ public class IncrementalIngester extends
protected static class DeleteInfo
{
protected String uriValue;
+ protected String uriHashValue;
protected String outputVersion;
- public DeleteInfo(String uriValue, String outputVersion)
+ public DeleteInfo(String uriValue, String uriHashValue, String
outputVersion)
{
this.uriValue = uriValue;
+ this.uriHashValue = uriHashValue;
this.outputVersion = outputVersion;
}
@@ -2364,6 +2380,11 @@ public class IncrementalIngester extends
return uriValue;
}
+ public String getURIHash()
+ {
+ return uriHashValue;
+ }
+
public String getOutputVersion()
{
return outputVersion;
@@ -3325,7 +3346,7 @@ public class IncrementalIngester extends
// But, since we need to insure that any given URI is only worked on by
one thread at a time, use critical sections
// to block the rare case that multiple threads try to work on the same
URI.
- String[] lockArray =
computeLockArray(documentURI,oldURI,outputConnectionName);
+ String[] lockArray =
computeLockArray(documentURIHash,oldURIHash,outputConnectionName);
lockManager.enterLocks(null,null,lockArray);
try
{
