Author: kwright
Date: Fri Nov 1 07:12:13 2013
New Revision: 1537836
URL: http://svn.apache.org/r1537836
Log:
Probable fix for CONNECTORS-798. Users of the JDBC connector, please try this
out.
Modified:
manifoldcf/trunk/CHANGES.txt
manifoldcf/trunk/connectors/jdbc/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jdbc/JDBCConnection.java
manifoldcf/trunk/connectors/jdbc/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jdbc/JDBCConnector.java
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/CharacterInput.java
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/NullCharacterInput.java
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/TempFileCharacterInput.java
Modified: manifoldcf/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1537836&r1=1537835&r2=1537836&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Fri Nov 1 07:12:13 2013
@@ -3,6 +3,11 @@ $Id$
======================= 1.5-dev =====================
+CONNECTORS-798: Bring JDBC connector support for CLOBs into the
+modern era. This also required extension of the CharacterInputFile
+paradigm slightly - a new method to find the utf8 byte length was needed.
+(Karl Wright)
+
CONNECTORS-797: Bad query when maxcount was exceeded in the
database table for the jobstatus screen.
(Graeme Seaton, Karl Wright)
Modified:
manifoldcf/trunk/connectors/jdbc/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jdbc/JDBCConnection.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/jdbc/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jdbc/JDBCConnection.java?rev=1537836&r1=1537835&r2=1537836&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/jdbc/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jdbc/JDBCConnection.java
(original)
+++
manifoldcf/trunk/connectors/jdbc/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jdbc/JDBCConnection.java
Fri Nov 1 07:12:13 2013
@@ -451,9 +451,8 @@ public class JDBCConnection
else if (isCLOB(rsmd,colnum))
{
Clob clob = getCLOB(rs,colnum);
- // Note well: we have not figured out how to handle characters
outside of ASCII!
if (clob != null)
- value = new TempFileInput(clob.getAsciiStream(),clob.length());
+ value = new
TempFileCharacterInput(clob.getCharacterStream(),clob.length());
}
else
{
@@ -558,42 +557,50 @@ public class JDBCConnection
// letting database do lame conversion!
ps.setString(i+1, value);
}
- if (x instanceof BinaryInput)
+ else if (x instanceof BinaryInput)
{
BinaryInput value = (BinaryInput)x;
+ ps.setBinaryStream(i+1,value.getStream(),value.getLength());
+ // Hopefully with the introduction of CharacterInput below, this
hackery is no longer needed.
// System.out.println("Blob length on write =
"+Long.toString(value.getLength()));
// The oracle driver does a binary conversion to base 64 when
writing data
// into a clob column using a binary stream operator. Since at this
// point there is no way to distinguish the two, and since our tests
use CLOB,
// this code doesn't work for them.
// So, for now, use the ascii stream method.
- //ps.setBinaryStream(i+1,value.getStream(),(int)value.getLength());
- ps.setAsciiStream(i+1,value.getStream(),(int)value.getLength());
+ //ps.setAsciiStream(i+1,value.getStream(),value.getLength());
+ }
+ else if (x instanceof CharacterInput)
+ {
+ CharacterInput value = (CharacterInput)x;
+
ps.setCharacterStream(i+1,value.getStream(),value.getCharacterLength());
}
- if (x instanceof java.util.Date)
+ else if (x instanceof java.util.Date)
{
ps.setDate(i+1,new java.sql.Date(((java.util.Date)x).getTime()));
}
- if (x instanceof Long)
+ else if (x instanceof Long)
{
ps.setLong(i+1,((Long)x).longValue());
}
- if (x instanceof TimeMarker)
+ else if (x instanceof TimeMarker)
{
ps.setTimestamp(i+1,new java.sql.Timestamp(((Long)x).longValue()));
}
- if (x instanceof Double)
+ else if (x instanceof Double)
{
ps.setDouble(i+1,((Double)x).doubleValue());
}
- if (x instanceof Integer)
+ else if (x instanceof Integer)
{
ps.setInt(i+1,((Integer)x).intValue());
}
- if (x instanceof Float)
+ else if (x instanceof Float)
{
ps.setFloat(i+1,((Float)x).floatValue());
}
+ else
+ throw new ManifoldCFException("Unknown data type:
"+x.getClass().getName());
}
}
}
@@ -615,6 +622,11 @@ public class JDBCConnection
BinaryInput value = (BinaryInput)x;
value.doneWithStream();
}
+ else if (x instanceof CharacterInput)
+ {
+ CharacterInput value = (CharacterInput)x;
+ value.doneWithStream();
+ }
}
}
}
Modified:
manifoldcf/trunk/connectors/jdbc/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jdbc/JDBCConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/jdbc/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jdbc/JDBCConnector.java?rev=1537836&r1=1537835&r2=1537836&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/jdbc/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jdbc/JDBCConnector.java
(original)
+++
manifoldcf/trunk/connectors/jdbc/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jdbc/JDBCConnector.java
Fri Nov 1 07:12:13 2013
@@ -591,6 +591,52 @@ public class JDBCConnector extends org.a
bi.discard();
}
}
+ else if (contents instanceof CharacterInput)
+ {
+ // An ingestion will take place for this document.
+ RepositoryDocument rd = new RepositoryDocument();
+
+ // Default content type is application/octet-stream for
binary data
+ if (contentType == null)
+ rd.setMimeType("text/plain; charset=utf-8");
+ else
+ rd.setMimeType(contentType);
+
+ applyAccessTokens(rd,version,spec);
+ applyMetadata(rd,row);
+
+ CharacterInput ci = (CharacterInput)contents;
+ try
+ {
+ // Read the stream
+ InputStream is = ci.getUtf8Stream();
+ try
+ {
+ rd.setBinary(is,ci.getUtf8StreamLength());
+ activities.ingestDocument(id, version, url, rd);
+ }
+ finally
+ {
+ is.close();
+ }
+ }
+ catch (java.net.SocketTimeoutException e)
+ {
+ throw new ManifoldCFException("Socket timeout reading
database data: "+e.getMessage(),e);
+ }
+ catch (InterruptedIOException e)
+ {
+ throw new ManifoldCFException("Interrupted:
"+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+ }
+ catch (IOException e)
+ {
+ throw new ManifoldCFException("Error reading database
data: "+e.getMessage(),e);
+ }
+ finally
+ {
+ ci.discard();
+ }
+ }
else
{
// Turn it into a string, and then into a stream
Modified:
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/CharacterInput.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/CharacterInput.java?rev=1537836&r1=1537835&r2=1537836&view=diff
==============================================================================
---
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/CharacterInput.java
(original)
+++
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/CharacterInput.java
Fri Nov 1 07:12:13 2013
@@ -76,6 +76,10 @@ public abstract class CharacterInput
public abstract InputStream getUtf8Stream()
throws ManifoldCFException;
+ /** Get binary UTF8 stream length directly */
+ public abstract long getUtf8StreamLength()
+ throws ManifoldCFException;
+
/** Transfer to a new object; this causes the current object to become
"already discarded" */
public abstract CharacterInput transfer();
Modified:
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/NullCharacterInput.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/NullCharacterInput.java?rev=1537836&r1=1537835&r2=1537836&view=diff
==============================================================================
---
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/NullCharacterInput.java
(original)
+++
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/NullCharacterInput.java
Fri Nov 1 07:12:13 2013
@@ -70,6 +70,14 @@ public class NullCharacterInput extends
return new ByteArrayInputStream(new byte[]{});
}
+ /** Get binary UTF8 stream length directly */
+ @Override
+ public long getUtf8StreamLength()
+ throws ManifoldCFException
+ {
+ return 0L;
+ }
+
/** Transfer to a new object; this causes the current object to become
"already discarded" */
@Override
public CharacterInput transfer()
Modified:
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/TempFileCharacterInput.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/TempFileCharacterInput.java?rev=1537836&r1=1537835&r2=1537836&view=diff
==============================================================================
---
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/TempFileCharacterInput.java
(original)
+++
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/TempFileCharacterInput.java
Fri Nov 1 07:12:13 2013
@@ -37,13 +37,24 @@ public class TempFileCharacterInput exte
protected final static int CHUNK_SIZE = 65536;
- /** Construct from a length-delimited reader.
+ /** Construct from a non-length-delimited reader.
*@param is is a reader to transfer from, to the end of the data. This will,
as a side effect, also calculate the character length
* and hash value for the data.
*/
public TempFileCharacterInput(Reader is)
throws ManifoldCFException
{
+ this(is,-1L);
+ }
+
+ /** Construct from a length-delimited reader.
+ *@param is is a reader to transfer from, to the end of the data. This will,
as a side effect, also calculate the character length
+ * and hash value for the data.
+ *@param length is the length limit to transfer, or -1 if no limit
+ */
+ public TempFileCharacterInput(Reader is, long length)
+ throws ManifoldCFException
+ {
super();
try
{
@@ -68,7 +79,13 @@ public class TempFileCharacterInput exte
long totalMoved = 0;
while (true)
{
- int moveAmount = CHUNK_SIZE;
+ int moveAmount;
+ if (length == -1L || length-totalMoved > CHUNK_SIZE)
+ moveAmount = CHUNK_SIZE;
+ else
+ moveAmount = (int)(length-totalMoved);
+ if (moveAmount == 0)
+ break;
// Read character data in 64K chunks
int readsize = is.read(buffer,0,moveAmount);
if (readsize == -1)
@@ -152,6 +169,16 @@ public class TempFileCharacterInput exte
return null;
}
+ /** Get binary UTF8 stream length directly */
+ @Override
+ public long getUtf8StreamLength()
+ throws ManifoldCFException
+ {
+ if (file != null)
+ return file.length();
+ return 0L;
+ }
+
@Override
protected void openStream()
throws ManifoldCFException
