Author: kwright
Date: Sat Feb 22 13:22:33 2014
New Revision: 1570837
URL: http://svn.apache.org/r1570837
Log:
Redefine RepositoryDocument security in a way that should be better compatible
with future additions
Modified:
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java
Modified:
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java?rev=1570837&r1=1570836&r2=1570837&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
(original)
+++
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
Sat Feb 22 13:22:33 2014
@@ -25,6 +25,7 @@ import java.io.InputStream;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Iterator;
import java.util.concurrent.TimeUnit;
import org.apache.http.conn.ClientConnectionManager;
@@ -404,6 +405,29 @@ public class ElasticSearchConnector exte
return null;
}
+ /** Convert an unqualified ACL to qualified form.
+ * @param acl is the initial, unqualified ACL.
+ * @param authorityNameString is the name of the governing authority for this
document's acls, or null if none.
+ * @param activities is the activities object, so we can report what's
happening.
+ * @return the modified ACL.
+ */
+ protected static String[] convertACL(String[] acl, String
authorityNameString, IOutputAddActivity activities)
+ throws ManifoldCFException
+ {
+ if (acl != null)
+ {
+ String[] rval = new String[acl.length];
+ int i = 0;
+ while (i < rval.length)
+ {
+ rval[i] = activities.qualifyAccessToken(authorityNameString,acl[i]);
+ i++;
+ }
+ return rval;
+ }
+ return new String[0];
+ }
+
@Override
public int addOrReplaceDocument(String documentURI, String outputDescription,
RepositoryDocument document, String authorityNameString,
@@ -413,11 +437,40 @@ public class ElasticSearchConnector exte
HttpClient client = getSession();
ElasticSearchConfig config = getConfigParameters(null);
InputStream inputStream = document.getBinaryStream();
+ // For ES, we have to have fixed fields only; nothing else is possible b/c
we don't have
+ // default field values.
+ String[] acls = null;
+ String[] denyAcls = null;
+ String[] shareAcls = null;
+ String[] shareDenyAcls = null;
+ Iterator<String> a = document.securityTypesIterator();
+ while (a.hasNext())
+ {
+ String securityType = a.next();
+ String[] convertedAcls =
convertACL(document.getSecurityACL(securityType),authorityNameString,activities);
+ String[] convertedDenyAcls =
convertACL(document.getSecurityDenyACL(securityType),authorityNameString,activities);
+ if (securityType.equals(RepositoryDocument.SECURITY_TYPE_DOCUMENT))
+ {
+ acls = convertedAcls;
+ denyAcls = convertedDenyAcls;
+ }
+ else if (securityType.equals(RepositoryDocument.SECURITY_TYPE_SHARE))
+ {
+ shareAcls = convertedAcls;
+ shareDenyAcls = convertedDenyAcls;
+ }
+ else
+ {
+ // Don't know how to deal with it
+ return DOCUMENTSTATUS_REJECTED;
+ }
+ }
+
long startTime = System.currentTimeMillis();
ElasticSearchIndex oi = new ElasticSearchIndex(client, config);
try
{
- oi.execute(documentURI, document, inputStream);
+ oi.execute(documentURI, document, inputStream, acls, denyAcls,
shareAcls, shareDenyAcls);
if (oi.getResult() != Result.OK)
return DOCUMENTSTATUS_REJECTED;
return DOCUMENTSTATUS_ACCEPTED;
Modified:
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java?rev=1570837&r1=1570836&r2=1570837&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
(original)
+++
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
Sat Feb 22 13:22:33 2014
@@ -57,14 +57,23 @@ public class ElasticSearchIndex extends
private class IndexRequestEntity implements HttpEntity
{
- private RepositoryDocument document;
- private InputStream inputStream;
+ private final RepositoryDocument document;
+ private final InputStream inputStream;
+ private final String[] acls;
+ private final String[] denyAcls;
+ private final String[] shareAcls;
+ private final String[] shareDenyAcls;
- public IndexRequestEntity(RepositoryDocument document, InputStream
inputStream)
+ public IndexRequestEntity(RepositoryDocument document, InputStream
inputStream,
+ String[] acls, String[] denyAcls, String[] shareAcls, String[]
shareDenyAcls)
throws ManifoldCFException
{
this.document = document;
this.inputStream = inputStream;
+ this.acls = acls;
+ this.denyAcls = denyAcls;
+ this.shareAcls = shareAcls;
+ this.shareDenyAcls = shareDenyAcls;
}
@Override
@@ -109,16 +118,8 @@ public class ElasticSearchIndex extends
needComma = writeField(pw, needComma, fieldName, fieldValues);
}
- needComma = writeACLs(pw, needComma, "document", document.getACL(),
document.getDenyACL());
- needComma = writeACLs(pw, needComma, "share", document.getShareACL(),
document.getShareDenyACL());
-
- int directoryCount = document.countDirectoryACLs();
- int q = 0;
- while (q < directoryCount)
- {
- needComma = writeACLs(pw, needComma, "directory_" + q,
document.getDirectoryACL(q), document.getDirectoryDenyACL(q));
- q++;
- }
+ needComma = writeACLs(pw, needComma, "document", acls, denyAcls);
+ needComma = writeACLs(pw, needComma, "share", shareAcls,
shareDenyAcls);
if(inputStream!=null){
if(needComma){
@@ -254,7 +255,8 @@ public class ElasticSearchIndex extends
*@return false to indicate that the document was rejected.
*/
public boolean execute(String documentURI, RepositoryDocument document,
- InputStream inputStream) throws ManifoldCFException, ServiceInterruption
+ InputStream inputStream, String[] acls, String[] denyAcls, String[]
shareAcls, String[] shareDenyAcls)
+ throws ManifoldCFException, ServiceInterruption
{
String idField;
try
@@ -268,7 +270,7 @@ public class ElasticSearchIndex extends
StringBuffer url = getApiUrl(config.getIndexType() + "/" + idField, false);
HttpPut put = new HttpPut(url.toString());
- put.setEntity(new IndexRequestEntity(document, inputStream));
+ put.setEntity(new IndexRequestEntity(document, inputStream, acls,
denyAcls, shareAcls, shareDenyAcls));
if (call(put) == false)
return false;
if ("true".equals(checkJson(jsonStatus)))
Modified:
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java?rev=1570837&r1=1570836&r2=1570837&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
(original)
+++
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
Sat Feb 22 13:22:33 2014
@@ -518,29 +518,29 @@ public class HttpPoster
if (maxDocumentLength != null && document.getBinaryLength() >
maxDocumentLength.longValue())
return false;
- // Convert the incoming acls to qualified forms
- String[] shareAcls =
convertACL(document.getShareACL(),authorityNameString,activities);
- String[] shareDenyAcls =
convertACL(document.getShareDenyACL(),authorityNameString,activities);
- String[] acls =
convertACL(document.getACL(),authorityNameString,activities);
- String[] denyAcls =
convertACL(document.getDenyACL(),authorityNameString,activities);
-
- Map<Integer,String[]> directoryAclsMap = new HashMap<Integer,String[]>();
- Map<Integer,String[]> directoryDenyAclsMap = new
HashMap<Integer,String[]>();
- int directoryCount = document.countDirectoryACLs();
- int q = 0;
- while (q < directoryCount)
- {
- String[] directoryAcls =
convertACL(document.getDirectoryACL(q),authorityNameString,activities);
- String[] directoryDenyAcls =
convertACL(document.getDirectoryDenyACL(q),authorityNameString,activities);
- directoryAclsMap.put(Integer.valueOf(q), directoryAcls);
- directoryDenyAclsMap.put(Integer.valueOf(q), directoryDenyAcls);
- q++;
+ // Convert the incoming acls that we know about to qualified forms, and
reject the document if
+ // we don't know how to deal with its acls
+ Map<String,String[]> aclsMap = new HashMap<String,String[]>();
+ Map<String,String[]> denyAclsMap = new HashMap<String,String[]>();
+
+ Iterator<String> aclTypes = document.securityTypesIterator();
+ while (aclTypes.hasNext())
+ {
+ String aclType = aclTypes.next();
+
aclsMap.put(aclType,convertACL(document.getSecurityACL(aclType),authorityNameString,activities));
+
denyAclsMap.put(aclType,convertACL(document.getSecurityDenyACL(aclType),authorityNameString,activities));
+
+ // Reject documents that have security we don't know how to deal with in
the Solr plugin!! Only safe thing to do.
+ if (!aclType.equals(RepositoryDocument.SECURITY_TYPE_DOCUMENT) &&
+ !aclType.equals(RepositoryDocument.SECURITY_TYPE_SHARE) &&
+ !aclType.startsWith(RepositoryDocument.SECURITY_TYPE_DIRECTORY_LEVEL))
+ return false;
}
try
{
IngestThread t = new
IngestThread(documentURI,document,arguments,keepAllMetadata,sourceTargets,
-
shareAcls,shareDenyAcls,directoryAclsMap,directoryDenyAclsMap,acls,denyAcls,commitWithin);
+ aclsMap,denyAclsMap,commitWithin);
try
{
t.start();
@@ -818,12 +818,8 @@ public class HttpPoster
protected final RepositoryDocument document;
protected final Map<String,List<String>> arguments;
protected final Map<String,List<String>> sourceTargets;
- protected final String[] shareAcls;
- protected final String[] shareDenyAcls;
- protected final Map<Integer,String[]> directoryAclsMap;
- protected final Map<Integer,String[]> directoryDenyAclsMap;
- protected final String[] acls;
- protected final String[] denyAcls;
+ protected final Map<String,String[]> aclsMap;
+ protected final Map<String,String[]> denyAclsMap;
protected final String commitWithin;
protected final boolean keepAllMetadata;
@@ -837,19 +833,16 @@ public class HttpPoster
public IngestThread(String documentURI, RepositoryDocument document,
Map<String, List<String>> arguments, boolean keepAllMetadata,
Map<String, List<String>> sourceTargets,
- String[] shareAcls, String[] shareDenyAcls, Map<Integer,String[]>
directoryAclsMap, Map<Integer,String[]> directoryDenyAclsMap, String[] acls,
String[] denyAcls, String commitWithin)
+ Map<String,String[]> aclsMap, Map<String,String[]> denyAclsMap,
+ String commitWithin)
{
super();
setDaemon(true);
this.documentURI = documentURI;
this.document = document;
this.arguments = arguments;
- this.shareAcls = shareAcls;
- this.shareDenyAcls = shareDenyAcls;
- this.directoryAclsMap = directoryAclsMap;
- this.directoryDenyAclsMap = directoryDenyAclsMap;
- this.acls = acls;
- this.denyAcls = denyAcls;
+ this.aclsMap = aclsMap;
+ this.denyAclsMap = denyAclsMap;
this.sourceTargets = sourceTargets;
this.commitWithin = commitWithin;
this.keepAllMetadata=keepAllMetadata;
@@ -912,18 +905,12 @@ public class HttpPoster
}
// Write the access token information
- writeACLs(out,"share",shareAcls,shareDenyAcls);
- writeACLs(out,"document",acls,denyAcls);
-
- int directoryCount = directoryAclsMap.size();
- int q = 0;
- while (q < directoryCount)
+ // Both maps have the same keys.
+ Iterator<String> typeIterator = aclsMap.keySet().iterator();
+ while (typeIterator.hasNext())
{
- Integer index = Integer.valueOf(q);
- String[] directoryAcls = directoryAclsMap.get(index);
- String[] directoryDenyAcls = directoryDenyAclsMap.get(index);
-
writeACLs(out,"directory_"+index.toString(),directoryAcls,directoryDenyAcls);
- q++;
+ String aclType = typeIterator.next();
+
writeACLs(out,aclType,aclsMap.get(aclType),denyAclsMap.get(aclType));
}
// Write the arguments
Modified:
manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java?rev=1570837&r1=1570836&r2=1570837&view=diff
==============================================================================
---
manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java
(original)
+++
manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java
Sat Feb 22 13:22:33 2014
@@ -36,16 +36,21 @@ public class RepositoryDocument
{
public static final String _rcsid = "@(#)$Id: RepositoryDocument.java 988245
2010-08-23 18:39:35Z kwright $";
+ // Security types
+ public final static String SECURITY_TYPE_DOCUMENT = "document";
+ public final static String SECURITY_TYPE_SHARE = "share";
+ public final static String SECURITY_TYPE_PARENT = "parent";
+ // Enumerated security type; add an integer to the end (deprecated)
+ public final static String SECURITY_TYPE_DIRECTORY_LEVEL = "directory_";
+
// Member variables.
protected InputStream binaryFieldData = null;
protected long binaryLength = 0;
- protected Map<String,Object> fields = new HashMap<String,Object>();
- protected Map<String,String[]> stringFields = new HashMap<String,String[]>();
- protected Map<String,Reader[]> readerFields = new HashMap<String,Reader[]>();
- protected Map<String,Date[]> dateFields = new HashMap<String,Date[]>();
- protected Security fileSecurity = new Security();
- protected Security shareSecurity = new Security();
- protected List<Security> directorySecurity = new ArrayList<Security>();
+ protected final Map<String,Object> fields = new HashMap<String,Object>();
+ protected final Map<String,String[]> stringFields = new
HashMap<String,String[]>();
+ protected final Map<String,Reader[]> readerFields = new
HashMap<String,Reader[]>();
+ protected final Map<String,Date[]> dateFields = new HashMap<String,Date[]>();
+ protected final Map<String,Security> securityLevels = new
HashMap<String,Security>();
protected String fileName = "docname";
protected String contentMimeType = "application/octet-stream";
protected Date createdDate = null;
@@ -122,95 +127,202 @@ public class RepositoryDocument
return contentMimeType;
}
+ /** Locate or create a specified security level.
+ *@param securityType is the security type.
+ */
+ protected Security getSecurityLevel(String securityType)
+ {
+ Security s = securityLevels.get(securityType);
+ if (s == null)
+ {
+ s = new Security();
+ securityLevels.put(securityType, s);
+ }
+ return s;
+ }
+
+ /** Enumerate the active security types for this document.
+ *@return an iterator over the security types.
+ */
+ public Iterator<String> securityTypesIterator()
+ {
+ return securityLevels.keySet().iterator();
+ }
+
+ /** Set security values for a given security type.
+ *@param securityType is the security type.
+ *@param acl is the acl.
+ *@param denyAcl is the deny acl.
+ */
+ public void setSecurity(String securityType, String[] acl, String[] denyAcl)
+ {
+ if (acl != null && denyAcl != null)
+ {
+ Security s = getSecurityLevel(securityType);
+ s.setACL(acl);
+ s.setDenyACL(denyAcl);
+ }
+ }
+
+ /** Set security acl for a given security type.
+ *@param securityType is the security type.
+ *@param acl is the acl;
+ */
+ public void setSecurityACL(String securityType, String[] acl)
+ {
+ if (acl != null)
+ {
+ Security s = getSecurityLevel(securityType);
+ s.setACL(acl);
+ }
+ }
+
+ /** Set security deny acl for a given security type.
+ *@param securityType is the security type.
+ *@param denyAcl is the deny acl.
+ */
+ public void setSecurityDenyACL(String securityType, String[] denyAcl)
+ {
+ if (denyAcl != null)
+ {
+ Security s = getSecurityLevel(securityType);
+ s.setDenyACL(denyAcl);
+ }
+ }
+
+ /** Get security acl for a given security type.
+ *@param securityType is the security type.
+ *@return the acl, which may be null.
+ */
+ public String[] getSecurityACL(String securityType)
+ {
+ Security s = securityLevels.get(securityType);
+ if (s != null)
+ return s.getACL();
+ return null;
+ }
+
+ /** Get security deny acl for a given security type.
+ *@param securityType is the security type.
+ *@return the acl, which may be null.
+ */
+ public String[] getSecurityDenyACL(String securityType)
+ {
+ Security s = securityLevels.get(securityType);
+ if (s != null)
+ return s.getDenyACL();
+ return null;
+ }
+
/** Set the document's "file" allow acls.
*@param acl is the allowed "file" access control token list for the document.
*/
+ @Deprecated
public void setACL(String[] acl)
{
- fileSecurity.setACL(acl);
+ setSecurityACL(SECURITY_TYPE_DOCUMENT,acl);
}
/** Get the document's "file" allow acl, if any.
*@return the allow access control token list for the document.
*/
+ @Deprecated
public String[] getACL()
{
- return fileSecurity.getACL();
+ return getSecurityACL(SECURITY_TYPE_DOCUMENT);
}
/** Set the document's "file" deny acl.
*@param acl is the "file" denied access control token list for the document.
*/
+ @Deprecated
public void setDenyACL(String[] acl)
{
- fileSecurity.setDenyACL(acl);
+ setSecurityDenyACL(SECURITY_TYPE_DOCUMENT,acl);
}
/** Get the document's deny acl, if any.
*@return the deny access control token list for the document.
*/
+ @Deprecated
public String[] getDenyACL()
{
- return fileSecurity.getDenyACL();
+ return getSecurityDenyACL(SECURITY_TYPE_DOCUMENT);
}
/** Set document's "share" acl. */
+ @Deprecated
public void setShareACL(String[] acl)
{
- shareSecurity.setACL(acl);
+ setSecurityACL(SECURITY_TYPE_SHARE,acl);
}
/** Get document's "share" acl. */
+ @Deprecated
public String[] getShareACL()
{
- return shareSecurity.getACL();
+ return getSecurityACL(SECURITY_TYPE_SHARE);
}
/** Set document's "share" deny acl. */
+ @Deprecated
public void setShareDenyACL(String[] acl)
{
- shareSecurity.setDenyACL(acl);
+ setSecurityDenyACL(SECURITY_TYPE_SHARE,acl);
}
/** Get document's "share" deny acl. */
+ @Deprecated
public String[] getShareDenyACL()
{
- return shareSecurity.getDenyACL();
+ return getSecurityDenyACL(SECURITY_TYPE_SHARE);
}
+ int directoryLevel = 0;
+
/** Clear all directory acls. */
+ @Deprecated
public void clearDirectoryACLs()
{
- directorySecurity.clear();
+ Iterator<String> securityIterator = securityLevels.keySet().iterator();
+ while (securityIterator.hasNext())
+ {
+ if (securityIterator.next().startsWith(SECURITY_TYPE_DIRECTORY_LEVEL))
+ securityIterator.remove();
+ }
+ directoryLevel = 0;
}
/** Get a count of directory security entries. */
+ @Deprecated
public int countDirectoryACLs()
{
- return directorySecurity.size();
+ return directoryLevel;
}
/** Add directory security entry */
+ @Deprecated
public void addDirectoryACLs(String[] allowACL, String[] denyACL)
{
- Security s = new Security();
- s.setACL(allowACL);
- s.setDenyACL(denyACL);
- directorySecurity.add(s);
+ if (allowACL != null && denyACL != null)
+ {
+ setSecurity(SECURITY_TYPE_DIRECTORY_LEVEL +
directoryLevel,allowACL,denyACL);
+ }
+ directoryLevel++;
}
/** Get directory security access acl */
+ @Deprecated
public String[] getDirectoryACL(int index)
{
- Security s = directorySecurity.get(index);
- return s.getACL();
+ return getSecurityACL(SECURITY_TYPE_DIRECTORY_LEVEL + index);
}
/** Get directory security deny acl */
+ @Deprecated
public String[] getDirectoryDenyACL(int index)
{
- Security s = directorySecurity.get(index);
- return s.getDenyACL();
+ return getSecurityDenyACL(SECURITY_TYPE_DIRECTORY_LEVEL + index);
}
/** Set the binary field.
