Author: kwright
Date: Tue Oct 21 09:32:07 2014
New Revision: 1633326
URL: http://svn.apache.org/r1633326
Log:
Pull up more CONNECTORS-1077 changes from trunk
Modified:
manifoldcf/branches/dev_1x/ (props changed)
manifoldcf/branches/dev_1x/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java
manifoldcf/branches/dev_1x/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
Propchange: manifoldcf/branches/dev_1x/
------------------------------------------------------------------------------
Merged /manifoldcf/trunk:r1633282,1633284,1633295
Modified:
manifoldcf/branches/dev_1x/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java?rev=1633326&r1=1633325&r2=1633326&view=diff
==============================================================================
---
manifoldcf/branches/dev_1x/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java
(original)
+++
manifoldcf/branches/dev_1x/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java
Tue Oct 21 09:32:07 2014
@@ -169,6 +169,9 @@ public class FileOutputConnector extends
FileOutputSpecs specs = new
FileOutputSpecs(getSpecNode(outputDescription.getSpecification()));;
StringBuffer path = new StringBuffer();
+ String errorCode = "OK";
+ String errorDesc = null;
+
try {
/*
* make file path
@@ -215,8 +218,9 @@ public class FileOutputConnector extends
}
else
{
-
activities.recordActivity(null,INGEST_ACTIVITY,null,documentURI,activities.CREATED_DIRECTORY,"Couldn't
create directory. '"+newPath+"'");
- throw new ManifoldCFException("Could not create directory
'"+newPath+"'. Permission issue?");
+ errorCode = activities.CREATED_DIRECTORY;
+ errorDesc = "Could not create directory '\"+newPath+\"'.
Permission issue?";
+ throw new ManifoldCFException(errorDesc);
}
}
// Directory successfully created!
@@ -247,8 +251,9 @@ public class FileOutputConnector extends
continue;
}
// Probably some other error
- activities.recordActivity(null, INGEST_ACTIVITY, new
Long(document.getBinaryLength()), documentURI, activities.EXCEPTION,
"Unexpected exception while creating the file.'" + outputPath + "'");
- throw new ManifoldCFException("Could not create file
'"+outputPath+"': "+e.getMessage(),e);
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Could not create file '" + outputPath + "': "
+e.getMessage();
+ throw new ManifoldCFException(errorDesc,e);
}
}
@@ -259,8 +264,9 @@ public class FileOutputConnector extends
FileChannel channel = output.getChannel();
FileLock lock = channel.tryLock();
if (lock == null){
- activities.recordActivity(null,INGEST_ACTIVITY, new
Long(document.getBinaryLength()), documentURI,activities.EXCEPTION,"Could not
lock file: '" + outputPath + "'");
- throw new ServiceInterruption("Could not lock file:
'"+outputPath+"'",null,1000L,-1L,10,false);
+ errorCode =
ServiceInterruption.class.getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Could not lock file: '"+outputPath+"'";
+ throw new ServiceInterruption(errorDesc,null,1000L,-1L,10,false);
}
@@ -292,24 +298,29 @@ public class FileOutputConnector extends
}
}
} catch (URISyntaxException e) {
- activities.recordActivity(null,INGEST_ACTIVITY,new
Long(document.getBinaryLength()), documentURI,activities.EXCEPTION,"Failed to
write document due to: " + e.getMessage());
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to write document due to: " + e.getMessage();
handleURISyntaxException(e);
return DOCUMENTSTATUS_REJECTED;
} catch (FileNotFoundException e) {
- activities.recordActivity(null,INGEST_ACTIVITY,new
Long(document.getBinaryLength()), documentURI,activities.EXCEPTION,"Failed to
write document due to: " + e.getMessage());
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to write document due to: " + e.getMessage();
handleFileNotFoundException(e);
return DOCUMENTSTATUS_REJECTED;
} catch (SecurityException e) {
- activities.recordActivity(null,INGEST_ACTIVITY,new
Long(document.getBinaryLength()), documentURI,activities.EXCEPTION,"Failed to
write document due to: " + e.getMessage());
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to write document due to: " + e.getMessage();
handleSecurityException(e);
return DOCUMENTSTATUS_REJECTED;
} catch (IOException e) {
- activities.recordActivity(null,INGEST_ACTIVITY,new
Long(document.getBinaryLength()), documentURI ,activities.EXCEPTION,"Failed to
write document due to: " + e.getMessage());
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to write document due to: " + e.getMessage();
handleIOException(e);
return DOCUMENTSTATUS_REJECTED;
+ } finally {
+ activities.recordActivity(null, INGEST_ACTIVITY, new
Long(document.getBinaryLength()), documentURI, errorCode, errorDesc);
}
- activities.recordActivity(null, INGEST_ACTIVITY, new
Long(document.getBinaryLength()), documentURI, "OK", null);
return DOCUMENTSTATUS_ACCEPTED;
}
@@ -361,6 +372,9 @@ public class FileOutputConnector extends
// Establish a session
getSession();
+ String errorCode = "OK";
+ String errorDesc = null;
+
FileOutputConfig config = getConfigParameters(null);
StringBuffer path = new StringBuffer();
@@ -432,27 +446,32 @@ public class FileOutputConnector extends
catch (FileNotFoundException e)
{
// Probably some other error
-
activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Couldn't
delete the file due to FileNotFoundException '"+outputPath+"':" +
e.getMessage());
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Couldn't delete the file due to:" + e.getMessage();
throw new ManifoldCFException("Could not zero out file
'"+outputPath+"': "+e.getMessage(),e);
}
}
// Just close it, to make a zero-length grave marker.
output.close();
} catch (URISyntaxException e) {
- activities.recordActivity(null,REMOVE_ACTIVITY,null,
documentURI,activities.EXCEPTION,"Failed to delete document due to: " +
e.getMessage());
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to delete document due to: " + e.getMessage();
handleURISyntaxException(e);
} catch (FileNotFoundException e) {
- activities.recordActivity(null,REMOVE_ACTIVITY,null,
documentURI,activities.EXCEPTION,"Failed to delete document due to: " +
e.getMessage());
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to delete document due to: " + e.getMessage();
handleFileNotFoundException(e);
} catch (SecurityException e) {
- activities.recordActivity(null,REMOVE_ACTIVITY,null,
documentURI,activities.EXCEPTION,"Failed to delete document due to: " +
e.getMessage());
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to delete document due to: " + e.getMessage();
handleSecurityException(e);
} catch (IOException e) {
- activities.recordActivity(null,REMOVE_ACTIVITY,null, documentURI
,activities.EXCEPTION,"Failed to delete document due to: " + e.getMessage());
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to delete document due to: " + e.getMessage();
handleIOException(e);
+ } finally {
+ activities.recordActivity(null, REMOVE_ACTIVITY, null, documentURI,
errorCode, errorDesc);
}
-
- activities.recordActivity(null, REMOVE_ACTIVITY, null, documentURI, "OK",
null);
}
/** Obtain the name of the form check javascript method to call.
Modified:
manifoldcf/branches/dev_1x/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java?rev=1633326&r1=1633325&r2=1633326&view=diff
==============================================================================
---
manifoldcf/branches/dev_1x/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
(original)
+++
manifoldcf/branches/dev_1x/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
Tue Oct 21 09:32:07 2014
@@ -290,7 +290,7 @@ public class HDFSOutputConnector extends
activities.recordActivity(startTime, INGEST_ACTIVITY, new
Long(document.getBinaryLength()), documentURI, "OK", null);
return DOCUMENTSTATUS_ACCEPTED;
} catch (URISyntaxException e) {
- activities.recordActivity(null,INGEST_ACTIVITY,new
Long(document.getBinaryLength()),documentURI,activities.EXCEPTION,"Failed to
write document due to: " + e.getMessage());
+ activities.recordActivity(null,INGEST_ACTIVITY,new
Long(document.getBinaryLength()),documentURI,e.getClass().getSimpleName().toUpperCase(Locale.ROOT),"Failed
to write document due to: " + e.getMessage());
handleURISyntaxException(e);
return DOCUMENTSTATUS_REJECTED;
}
@@ -324,10 +324,10 @@ public class HDFSOutputConnector extends
deleteFile(path,activities,documentURI);
activities.recordActivity(startTime, REMOVE_ACTIVITY, null, documentURI,
"OK", null);
} catch (JSONException e) {
-
activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Failed
to delete document due to: " + e.getMessage());
+
activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,e.getClass().getSimpleName().toUpperCase(Locale.ROOT),"Failed
to delete document due to: " + e.getMessage());
handleJSONException(e);
} catch (URISyntaxException e) {
-
activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Failed
to delete document due to: " + e.getMessage());
+
activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,e.getClass().getSimpleName().toUpperCase(Locale.ROOT),"Failed
to delete document due to: " + e.getMessage());
handleURISyntaxException(e);
}
}
@@ -675,23 +675,33 @@ public class HDFSOutputConnector extends
protected void createFile(Path path, InputStream input,IOutputAddActivity
activities, String documentURI)
throws ManifoldCFException, ServiceInterruption {
CreateFileThread t = new CreateFileThread(getSession(), path, input);
+ String errorCode = null;
+ String errorDesc = null;
try {
t.start();
t.finishUp();
} catch (InterruptedException e) {
t.interrupt();
-
activities.recordActivity(null,INGEST_ACTIVITY,null,documentURI,activities.EXCEPTION,"Failed
to write document due to: " + e.getMessage());
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to write document due to: " + e.getMessage();
throw new ManifoldCFException("Interrupted:
"+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
} catch (java.net.SocketTimeoutException e) {
-
activities.recordActivity(null,INGEST_ACTIVITY,null,documentURI,activities.EXCEPTION,"Failed
to write document due to: " + e.getMessage());
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to write document due to: " + e.getMessage();
handleIOException(e);
} catch (InterruptedIOException e) {
t.interrupt();
-
activities.recordActivity(null,INGEST_ACTIVITY,null,documentURI,activities.EXCEPTION,"Failed
to write document due to: " + e.getMessage());
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to write document due to: " + e.getMessage();
handleIOException(e);
} catch (IOException e) {
-
activities.recordActivity(null,INGEST_ACTIVITY,null,documentURI,activities.EXCEPTION,"Failed
to write document due to: " + e.getMessage());
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to write document due to: " + e.getMessage();
handleIOException(e);
+ } finally {
+ if(errorCode != null & errorDesc != null){
+
activities.recordActivity(null,INGEST_ACTIVITY,null,documentURI,errorCode,errorDesc);
+ }
}
}
@@ -734,23 +744,33 @@ public class HDFSOutputConnector extends
throws ManifoldCFException, ServiceInterruption {
// Establish a session
DeleteFileThread t = new DeleteFileThread(getSession(),path);
+ String errorCode = null;
+ String errorDesc = null;
try {
t.start();
t.finishUp();
} catch (InterruptedException e) {
- t.interrupt();
-
activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Failed
to delete document due to: " + e.getMessage());
- throw new ManifoldCFException("Interrupted:
"+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+ t.interrupt();
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to write document due to: " + e.getMessage();
+ throw new ManifoldCFException("Interrupted:
"+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
} catch (java.net.SocketTimeoutException e) {
-
activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Failed
to delete document due to: " + e.getMessage());
- handleIOException(e);
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to write document due to: " + e.getMessage();
+ handleIOException(e);
} catch (InterruptedIOException e) {
- t.interrupt();
-
activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Failed
to delete document due to: " + e.getMessage());
- handleIOException(e);
+ t.interrupt();
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to write document due to: " + e.getMessage();
+ handleIOException(e);
} catch (IOException e) {
-
activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Failed
to delete document due to: " + e.getMessage());
- handleIOException(e);
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to write document due to: " + e.getMessage();
+ handleIOException(e);
+ } finally {
+ if(errorCode != null & errorDesc != null){
+
activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,errorCode,errorDesc);
+ }
}
}
Modified:
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java?rev=1633326&r1=1633325&r2=1633326&view=diff
==============================================================================
---
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
(original)
+++
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
Tue Oct 21 09:32:07 2014
@@ -536,8 +536,11 @@ public class HttpPoster
Logging.ingest.debug("indexPost(): '" + documentURI + "'");
// If the document is too long, reject it.
- if (maxDocumentLength != null && document.getBinaryLength() >
maxDocumentLength.longValue())
+ if (maxDocumentLength != null && document.getBinaryLength() >
maxDocumentLength.longValue()){
+
activities.recordActivity(null,SolrConnector.INGEST_ACTIVITY,null,documentURI,activities.EXCLUDED_LENGTH,"Solr
connector rejected document due to its big size.
('"+document.getBinaryLength()+"')");
return false;
+ }
+
// Convert the incoming acls that we know about to qualified forms, and
reject the document if
// we don't know how to deal with its acls
@@ -554,8 +557,11 @@ public class HttpPoster
// Reject documents that have security we don't know how to deal with in
the Solr plugin!! Only safe thing to do.
if (!aclType.equals(RepositoryDocument.SECURITY_TYPE_DOCUMENT) &&
!aclType.equals(RepositoryDocument.SECURITY_TYPE_SHARE) &&
- !aclType.startsWith(RepositoryDocument.SECURITY_TYPE_PARENT))
- return false;
+ !aclType.startsWith(RepositoryDocument.SECURITY_TYPE_PARENT)){
+
activities.recordActivity(null,SolrConnector.INGEST_ACTIVITY,null,documentURI,activities.UNKNOWN_SECURITY,"Solr
connector rejected document that has security info which is unknown.");
+ return false;
+ }
+
}
try
@@ -940,9 +946,9 @@ public class HttpPoster
(activityDetails.toLowerCase(Locale.ROOT).indexOf("broken pipe")
!= -1 ||
activityDetails.toLowerCase(Locale.ROOT).indexOf("connection
reset") != -1 ||
activityDetails.toLowerCase(Locale.ROOT).indexOf("target
server failed to respond") != -1))
- activityCode = "SOLR REJECT";
+ activityCode = "SOLRREJECT";
else
- activityCode = "FAILED";
+ activityCode =
e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
// Rethrow; will interpret at a higher level
throw e;
@@ -966,7 +972,7 @@ public class HttpPoster
return;
activityStart = new Long(fullStartTime);
- activityCode = "IO ERROR";
+ activityCode =
ioe.getClass().getSimpleName().toUpperCase(Locale.ROOT);
activityDetails = ioe.getMessage();
// Log the error
@@ -1331,7 +1337,7 @@ public class HttpPoster
catch (SolrServerException e)
{
activityStart = new Long(fullStartTime);
- activityCode = "FAILED";
+ activityCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
activityDetails = e.getMessage() +
((e.getCause() != null)?": "+e.getCause().getMessage():"");
@@ -1340,7 +1346,7 @@ public class HttpPoster
catch (SolrException e)
{
activityStart = new Long(fullStartTime);
- activityCode = "FAILED";
+ activityCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
activityDetails = e.getMessage() +
((e.getCause() != null)?": "+e.getCause().getMessage():"");
@@ -1352,7 +1358,7 @@ public class HttpPoster
Logging.ingest.warn("Error deleting document:
"+ioe.getMessage(),ioe);
activityStart = new Long(fullStartTime);
- activityCode = "IO ERROR";
+ activityCode =
ioe.getClass().getSimpleName().toUpperCase(Locale.ROOT);
activityDetails = ioe.getMessage();
throw ioe;
