Author: kwright
Date: Wed Aug 26 19:03:55 2015
New Revision: 1697991
URL: http://svn.apache.org/r1697991
Log:
Add amazon s3 connector
Added:
manifoldcf/branches/CONNECTORS-1233/amazons3/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/authorities/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/amazons3/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/amazons3/AmazonS3Authority.java
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/amazons3/AmazonS3Config.java
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/amazons3/Messages.java
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/amazons3/ResponseException.java
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/connectors/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/connectors/common/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/connectors/common/amazons3/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/connectors/common/amazons3/S3Artifact.java
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/connectors/common/amazons3/XThreadBuffer.java
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/AmazonS3Config.java
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/AmazonS3Connector.java
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/AmazonS3DocumentProcessUtility.java
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/DocumentProcess.java
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/GenericDocumentProcess.java
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/Messages.java
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/ResponseException.java
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/TikaBasedDocumentProcess.java
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/native2ascii/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/native2ascii/org/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/native2ascii/org/apache/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/native2ascii/org/apache/manifoldcf/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/native2ascii/org/apache/manifoldcf/authorities/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/native2ascii/org/apache/manifoldcf/authorities/authorities/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/native2ascii/org/apache/manifoldcf/authorities/authorities/amazons3/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/native2ascii/org/apache/manifoldcf/authorities/authorities/amazons3/common_en_US.properties
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/amazons3/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/amazons3/common_en_US.properties
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/authorities/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/authorities/authorities/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/authorities/authorities/amazons3/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/authorities/authorities/amazons3/editConfiguration_amazons3.js
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/authorities/authorities/amazons3/editConfiguration_amazons3_proxy.html
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/authorities/authorities/amazons3/editConfiguration_amazons3_server.html
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/authorities/authorities/amazons3/viewConfiguration_amazons3.html
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/editConfiguration_amazons3.js
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/editConfiguration_amazons3_proxy.html
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/editConfiguration_amazons3_server.html
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/editSpecification_amazons3.js
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/editSpecification_buckets.html
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/viewConfiguration_amazons3.html
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/viewSpecification_amazons3.html
manifoldcf/branches/CONNECTORS-1233/amazons3/pom.xml
Added:
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/amazons3/AmazonS3Authority.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/amazons3/AmazonS3Authority.java?rev=1697991&view=auto
==============================================================================
---
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/amazons3/AmazonS3Authority.java
(added)
+++
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/amazons3/AmazonS3Authority.java
Wed Aug 26 19:03:55 2015
@@ -0,0 +1,593 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.manifoldcf.authorities.authorities.amazons3;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.manifoldcf.authorities.authorities.BaseAuthorityConnector;
+import org.apache.manifoldcf.authorities.interfaces.AuthorizationResponse;
+import org.apache.manifoldcf.authorities.system.Logging;
+import org.apache.manifoldcf.core.interfaces.ConfigNode;
+import org.apache.manifoldcf.core.interfaces.ConfigParams;
+import org.apache.manifoldcf.core.interfaces.IHTTPOutput;
+import org.apache.manifoldcf.core.interfaces.IPasswordMapperActivity;
+import org.apache.manifoldcf.core.interfaces.IPostParameters;
+import org.apache.manifoldcf.core.interfaces.IThreadContext;
+import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.AmazonServiceException;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.model.AccessControlList;
+import com.amazonaws.services.s3.model.Bucket;
+import com.amazonaws.services.s3.model.CanonicalGrantee;
+import com.amazonaws.services.s3.model.Grant;
+import com.amazonaws.services.s3.model.Grantee;
+import com.amazonaws.services.s3.model.Owner;
+
+/**
+ * Authority connector for Amazons3
+ * @author Kuhajeyan
+ *
+ */
+public class AmazonS3Authority extends
org.apache.manifoldcf.authorities.authorities.BaseAuthorityConnector {
+ private static final String TAB_NAME = "TabName";
+
+ protected long lastSessionFetch = -1L;
+
+ protected static final long timeToRelease = 300000L;
+
+ protected AmazonS3 amazonS3;
+
+ protected boolean connected = false;
+
+ protected String amazons3ProxyHost = null;
+
+ protected String amazons3ProxyPort = null;
+
+ protected String amazons3ProxyDomain = null;
+
+ protected String amazons3ProxyUserName = null;
+
+ protected String amazons3ProxyPassword = null;
+
+ protected String amazons3AwsAccessKey = null;
+
+ protected String amazons3AwsSecretKey = null;
+
+ public AmazonS3Authority() {
+
+ }
+
+ @Override
+ public void disconnect() throws ManifoldCFException {
+ amazons3AwsAccessKey = null;
+ amazons3AwsSecretKey = null;
+
+ amazons3ProxyHost = null;
+ amazons3ProxyPort = null;
+ amazons3ProxyDomain = null;
+ amazons3ProxyUserName = null;
+ amazons3ProxyPassword = null;
+ }
+
+ @Override
+ public void connect(ConfigParams configParams) {
+ super.connect(configParams);
+ // aws access and secret keys
+ amazons3AwsAccessKey = configParams
+ .getParameter(AmazonS3Config.AWS_ACCESS_KEY);
+ amazons3AwsSecretKey = configParams
+
.getObfuscatedParameter(AmazonS3Config.AWS_SECRET_KEY);
+
+ // proxy values
+ amazons3ProxyHost = configParams
+
.getParameter(AmazonS3Config.AMAZONS3_PROXY_HOST);
+ amazons3ProxyPort = configParams
+
.getParameter(AmazonS3Config.AMAZONS3_PROXY_PORT);
+ amazons3ProxyDomain = configParams
+
.getParameter(AmazonS3Config.AMAZONS3_PROXY_DOMAIN);
+ amazons3ProxyUserName = configParams
+
.getParameter(AmazonS3Config.AMAZONS3_PROXY_USERNAME);
+ amazons3ProxyPassword = configParams
+
.getObfuscatedParameter(AmazonS3Config.AMAZONS3_PROXY_PASSWORD);
+ }
+
+ /**
+ * Test the connection. Returns a string describing the connection
+ * integrity.
+ *
+ * @return the connection's status as a displayable string.
+ */
+ @Override
+ public String check() throws ManifoldCFException {
+ // connect with amazons3 client
+ Logging.authorityConnectors.info("Checking connection");
+
+ try {
+ // invokes the check thread
+ CheckThread checkThread = new CheckThread(getClient());
+ checkThread.start();
+ checkThread.join();
+ if (checkThread.getException() != null) {
+ Throwable thr = checkThread.getException();
+ return "Check exception: " + thr.getMessage();
+ }
+ return checkThread.getResult();
+ }
+ catch (InterruptedException ex) {
+ Logging.authorityConnectors.error(
+ "Error while checking connection", ex);
+ throw new ManifoldCFException(ex.getMessage(), ex,
+ ManifoldCFException.INTERRUPTED);
+ }
+
+ }
+
+ /**
+ * Get the Amazons3 client, relevant access keys should have been posted
+ * already
+ * @return
+ */
+ protected AmazonS3 getClient() {
+ if (amazonS3 == null) {
+ try {
+ BasicAWSCredentials awsCreds = new
BasicAWSCredentials(
+ amazons3AwsAccessKey,
amazons3AwsSecretKey);
+ amazonS3 = new AmazonS3Client(awsCreds);
+ }
+ catch (Exception e) {
+ Logging.authorityConnectors.error(
+ "Error while amazon s3
connectionr", e);
+
+ }
+ }
+ lastSessionFetch = System.currentTimeMillis();
+ return amazonS3;
+ }
+
+ @Override
+ public boolean isConnected() {
+ return amazonS3 != null && amazonS3.getS3AccountOwner() != null;
+ }
+
+ @Override
+ public void poll() throws ManifoldCFException {
+ if (lastSessionFetch == -1L) {
+ return;
+ }
+
+ long currentTime = System.currentTimeMillis();
+ if (currentTime >= lastSessionFetch + timeToRelease) {
+ amazonS3 = null;
+ lastSessionFetch = -1L;
+ }
+ }
+
+ private void fillInServerConfigurationMap(Map<String, Object> out,
+ IPasswordMapperActivity mapper, ConfigParams
parameters) {
+
+ String amazons3AccessKey = parameters
+ .getParameter(AmazonS3Config.AWS_ACCESS_KEY);
+ String amazons3SecretKey = parameters
+ .getParameter(AmazonS3Config.AWS_SECRET_KEY);
+
+ // default values
+ if (amazons3AccessKey == null)
+ amazons3AccessKey =
AmazonS3Config.AMAZONS3_AWS_ACCESS_KEY_DEFAULT;
+ if (amazons3SecretKey == null)
+ amazons3SecretKey =
AmazonS3Config.AMAZONS3_AWS_SECRET_KEY_DEFAULT;
+ else
+ amazons3SecretKey =
mapper.mapPasswordToKey(amazons3SecretKey);
+
+ // fill the map
+ out.put("AMAZONS3_AWS_ACCESS_KEY", amazons3AccessKey);
+ out.put("AMAZONS3_AWS_SECRET_KEY", amazons3SecretKey);
+ }
+
+ private void fillInProxyConfigurationMap(Map<String, Object> out,
+ IPasswordMapperActivity mapper, ConfigParams
parameters) {
+ String amazons3ProxyHost = parameters
+
.getParameter(AmazonS3Config.AMAZONS3_PROXY_HOST);
+ String amazons3ProxyPort = parameters
+
.getParameter(AmazonS3Config.AMAZONS3_PROXY_PORT);
+ String amazons3ProxyDomain = parameters
+
.getParameter(AmazonS3Config.AMAZONS3_PROXY_DOMAIN);
+ String amazons3ProxyUserName = parameters
+
.getParameter(AmazonS3Config.AMAZONS3_PROXY_USERNAME);
+ String amazons3ProxyPassword = parameters
+
.getObfuscatedParameter(AmazonS3Config.AMAZONS3_PROXY_PASSWORD);
+
+ if (amazons3ProxyHost == null)
+ amazons3ProxyHost =
AmazonS3Config.AMAZONS3_PROXY_HOST_DEFAULT;
+ if (amazons3ProxyPort == null)
+ amazons3ProxyPort =
AmazonS3Config.AMAZONS3_PROXY_PORT_DEFAULT;
+ if (amazons3ProxyDomain == null)
+ amazons3ProxyDomain =
AmazonS3Config.AMAZONS3_PROXY_DOMAIN_DEFAULT;
+ if (amazons3ProxyUserName == null)
+ amazons3ProxyUserName =
AmazonS3Config.AMAZONS3_PROXY_USERNAME_DEFAULT;
+ if (amazons3ProxyPassword == null)
+ amazons3ProxyPassword =
AmazonS3Config.AMAZONS3_PROXY_PASSWORD_DEFAULT;
+ else
+ amazons3ProxyPassword = mapper
+
.mapPasswordToKey(amazons3ProxyPassword);
+
+ // fill the map
+ out.put("AMAZONS3_PROXY_HOST", amazons3ProxyHost);
+ out.put("AMAZONS3_PROXY_PORT", amazons3ProxyPort);
+ out.put("AMAZONS3_PROXY_DOMAIN", amazons3ProxyDomain);
+ out.put("AMAZONS3_PROXY_USERNAME", amazons3ProxyUserName);
+ out.put("AMAZONS3_PROXY_PWD", amazons3ProxyPassword);
+ }
+
+ /**
+ * View configuration. This method is called in the body section of the
+ * connector's view configuration page. Its purpose is to present the
+ * connection information to the user. The coder can presume that the
HTML
+ * that is output from this configuration will be within appropriate
<html>
+ * and <body> tags.
+ *
+ * */
+ public void viewConfiguration(IThreadContext threadContext,
+ IHTTPOutput out, Locale locale, ConfigParams parameters)
+ throws ManifoldCFException, IOException {
+ Map<String, Object> paramMap = new HashMap<String, Object>();
+
+ // Fill in map from each tab
+ fillInServerConfigurationMap(paramMap, out, parameters);
+ fillInProxyConfigurationMap(paramMap, out, parameters);
+
+ Messages.outputResourceWithVelocity(out, locale,
AmazonS3Config.VIEW_CONFIG_FORWARD, paramMap);
+ }
+
+ /**
+ * Output the configuration header section. This method is called in the
+ * head section of the connector's configuration page. Its purpose is
to add
+ * the required tabs to the list, and to output any javascript methods
that
+ * might be needed by the configuration editing HTML.
+ * */
+ @Override
+ public void outputConfigurationHeader(IThreadContext threadContext,
+ IHTTPOutput out, Locale locale, ConfigParams parameters,
+ List<String> tabsArray) throws ManifoldCFException,
IOException {
+ // Add the Server tab
+ tabsArray.add(Messages.getString(locale,
+ AmazonS3Config.AMAZONS3_SERVER_TAB_PROPERTY));
+ // Add the Proxy tab
+ tabsArray.add(Messages.getString(locale,
+ AmazonS3Config.AMAZONS3_PROXY_TAB_PROPERTY));
+ // Map the parameters
+ Map<String, Object> paramMap = new HashMap<String, Object>();
+
+ // Fill in the parameters from each tab
+ fillInServerConfigurationMap(paramMap, out, parameters);
+ fillInProxyConfigurationMap(paramMap, out, parameters);
+
+ // Output the Javascript - only one Velocity template for all
tabs
+ Messages.outputResourceWithVelocity(out, locale,
+ AmazonS3Config.EDIT_CONFIG_HEADER_FORWARD,
paramMap);
+ }
+
+ @Override
+ public void outputConfigurationBody(IThreadContext threadContext,
+ IHTTPOutput out, Locale locale, ConfigParams parameters,
+ String tabName) throws ManifoldCFException, IOException
{
+
+ // Call the Velocity templates for each tab
+ Map<String, Object> paramMap = new HashMap<String, Object>();
+ // Set the tab name
+ paramMap.put(TAB_NAME, tabName);
+
+ // Fill in the parameters
+ fillInServerConfigurationMap(paramMap, out, parameters);
+ fillInProxyConfigurationMap(paramMap, out, parameters);
+
+ // Server tab
+ Messages.outputResourceWithVelocity(out, locale,
+ AmazonS3Config.EDIT_CONFIG_FORWARD_SERVER,
paramMap);
+ // Proxy tab
+ Messages.outputResourceWithVelocity(out, locale,
+ AmazonS3Config.EDIT_CONFIG_FORWARD_PROXY,
paramMap);
+
+ }
+
+ /**
+ * Process a configuration post. This method is called at the start of
the
+ * connector's configuration page, whenever there is a possibility that
form
+ * data for a connection has been posted. Its purpose is to gather form
+ * information and modify the configuration parameters accordingly. The
name
+ * of the posted form is "editconnection".
+ * */
+ @Override
+ public String processConfigurationPost(IThreadContext threadContext,
+ IPostParameters variableContext, Locale locale,
+ ConfigParams parameters) throws ManifoldCFException {
+ // server tab
+ String awsAccessKey =
variableContext.getParameter("aws_access_key");
+
+ if (awsAccessKey != null) {
+ parameters
+
.setParameter(AmazonS3Config.AWS_ACCESS_KEY, awsAccessKey);
+ }
+ String awsSecretKey =
variableContext.getParameter("aws_secret_key");
+ if (awsSecretKey != null) {
+ // set as obfuscated parameter
+
parameters.setObfuscatedParameter(AmazonS3Config.AWS_SECRET_KEY,
+ awsSecretKey);
+ }
+ Logging.authorityConnectors.info("Saved values for aws keys");
+
+ int i = 0;
+ while (i < parameters.getChildCount()) {
+ ConfigNode cn = parameters.getChild(i);
+ if (cn.getType().equals(AmazonS3Config.AWS_ACCESS_KEY)
+ ||
cn.getType().equals(AmazonS3Config.AWS_SECRET_KEY))
+ parameters.removeChild(i);
+ else
+ i++;
+ }
+
+ // proxy tab
+ String amazons3ProxyHost = variableContext
+ .getParameter("amazons3_proxy_host");
+ if (amazons3ProxyHost != null) {
+
parameters.setParameter(AmazonS3Config.AMAZONS3_PROXY_HOST,
+ amazons3ProxyHost);
+ }
+ String amazons3ProxyPort = variableContext
+ .getParameter("amazons3_proxy_port");
+ if (amazons3ProxyPort != null) {
+
parameters.setParameter(AmazonS3Config.AMAZONS3_PROXY_PORT,
+ amazons3ProxyPort);
+ }
+ String amazons3ProxyDomain = variableContext
+ .getParameter("amazons3_proxy_domain");
+ if (amazons3ProxyDomain != null) {
+
parameters.setParameter(AmazonS3Config.AMAZONS3_PROXY_DOMAIN,
+ amazons3ProxyDomain);
+ }
+ String amazons3ProxyUserName = variableContext
+ .getParameter("amazons3_proxy_username");
+ if (amazons3ProxyUserName != null) {
+
parameters.setParameter(AmazonS3Config.AMAZONS3_PROXY_USERNAME,
+ amazons3ProxyUserName);
+ }
+ String amazons3ProxyPassword = variableContext
+ .getParameter("amazons3_proxy_pwd");
+ if (amazons3ProxyPassword != null) {
+ // set as obfuscated parameter
+ parameters.setObfuscatedParameter(
+ AmazonS3Config.AMAZONS3_PROXY_USERNAME,
+ amazons3ProxyUserName);
+ }
+
+ return null;
+ }
+
+ @Override
+ public AuthorizationResponse getAuthorizationResponse(String userName)
+ throws ManifoldCFException {
+
+ HashMap<String, Set<Grant>> checkUserExists =
checkUserExists(userName);
+ if (isUserAvailable(userName, checkUserExists.values())) {
+ return new AuthorizationResponse(new String[] {
userName },
+ AuthorizationResponse.RESPONSE_OK);
+ }
+ return RESPONSE_USERNOTFOUND;
+ }
+
+ private boolean isUserAvailable(String userName,
+ Collection<Set<Grant>> collection) {
+ String[] users = getUsers(collection);
+ return Arrays.asList(users).contains(userName);
+ }
+
+ private String[] getUsers(Collection<Set<Grant>> collection) {
+ Set<String> users = new HashSet<String>();// no duplicates
+ for (Collection c : collection) {
+ Set<Grant> c1 = (Set<Grant>) c;
+ for (Grant grant : c1) {
+ if (grant != null && grant.getGrantee() !=
null) {
+ Grantee grantee = grant.getGrantee();
+
+ if (grantee instanceof
CanonicalGrantee) {
+ users.add(((CanonicalGrantee)
grantee).getDisplayName());
+ }
+ else {
+
users.add(grantee.getIdentifier());
+ }
+ }
+ }
+ }
+
+ return users.toArray(new String[users.size()]);
+ }
+
+ private HashMap<String, Set<Grant>> checkUserExists(String userName)
+ throws ManifoldCFException {
+ GrantsThread t = new GrantsThread(getClient());
+ try {
+ t.start();
+ t.finishUp();
+ return t.getResult();
+ }
+ catch (InterruptedException e) {
+ t.interrupt();
+ throw new ManifoldCFException("Interrupted: " +
e.getMessage(), e,
+ ManifoldCFException.INTERRUPTED);
+ }
+ catch (java.net.SocketTimeoutException e) {
+ handleIOException(e);
+ }
+ catch (InterruptedIOException e) {
+ t.interrupt();
+ handleIOException(e);
+ }
+ catch (IOException e) {
+ handleIOException(e);
+ }
+ catch (ResponseException e) {
+ handleResponseException(e);
+ }
+ return null;
+ }
+
+ /**
+ * Obtain the default access tokens for a given user name.
+ * @param userName is the user name or identifier.
+ * @return the default response tokens, presuming that the connect
method
+ * fails.
+ */
+ @Override
+ public AuthorizationResponse getDefaultAuthorizationResponse(String
userName) {
+ return RESPONSE_UNREACHABLE;
+ }
+
+ private static void handleIOException(IOException e)
+ throws ManifoldCFException {
+ if (!(e instanceof java.net.SocketTimeoutException)
+ && (e instanceof InterruptedIOException)) {
+ throw new ManifoldCFException("Interrupted: " +
e.getMessage(), e,
+ ManifoldCFException.INTERRUPTED);
+ }
+ Logging.authorityConnectors.warn(
+ "JIRA: IO exception: " + e.getMessage(), e);
+ throw new ManifoldCFException("IO exception: " +
e.getMessage(), e);
+ }
+
+ private static void handleResponseException(ResponseException e)
+ throws ManifoldCFException {
+ throw new ManifoldCFException("Response exception: " +
e.getMessage(),
+ e);
+ }
+
+ protected static class GrantsThread extends Thread {
+
+ protected Throwable exception = null;
+
+ protected boolean result = false;
+
+ protected AmazonS3 amazonS3 = null;
+
+ private HashMap<String, Set<Grant>> grants;
+
+ public GrantsThread(AmazonS3 amazonS3) {
+ super();
+ this.amazonS3 = amazonS3;
+ setDaemon(true);
+ grants = new HashMap<String, Set<Grant>>();
+ }
+
+ public void finishUp() throws InterruptedException, IOException,
+ ResponseException {
+ join();
+ Throwable thr = exception;
+ if (thr != null) {
+ if (thr instanceof IOException) {
+ throw (IOException) thr;
+ }
+ else if (thr instanceof ResponseException) {
+ throw (ResponseException) thr;
+ }
+ else if (thr instanceof RuntimeException) {
+ throw (RuntimeException) thr;
+ }
+ else {
+ throw (Error) thr;
+ }
+ }
+ }
+
+ @Override
+ public void run() {
+
+ List<Bucket> listBuckets = amazonS3.listBuckets();
+ for (Bucket bucket : listBuckets) {
+ AccessControlList bucketAcl =
amazonS3.getBucketAcl(bucket
+ .getName());
+
+ if (bucketAcl != null)
+ grants.put(bucket.getName(),
bucketAcl.getGrants());
+ }
+ }
+
+ public HashMap<String, Set<Grant>> getResult() {
+ return grants;
+ }
+
+ }
+
+ protected static class CheckThread extends Thread {
+ protected String result = "Unknown";
+
+ protected AmazonS3 s3 = null;
+
+ protected Throwable exception = null;
+
+ public CheckThread(AmazonS3 s3) {
+ this.s3 = s3;
+ }
+
+ public String getResult() {
+ return result;
+ }
+
+ public Throwable getException() {
+ return exception;
+ }
+
+ @Override
+ public void run() {
+ try {
+ if (s3 != null) {
+ Owner s3AccountOwner =
s3.getS3AccountOwner();
+ if (s3AccountOwner != null) {
+ result =
StringUtils.isNotEmpty(s3AccountOwner
+
.getDisplayName()) ? "Connection OK"
+ : "Connection
Failed";
+ }
+
+ }
+ }
+ catch (AmazonServiceException e) {
+ result = "Connection Failed : " +
e.getMessage();
+ exception = e;
+
+ Logging.authorityConnectors.error(e);
+ }
+ catch (AmazonClientException e) {
+ result = "Connection Failed : " +
e.getMessage();
+ exception = e;
+
+ Logging.authorityConnectors.error(e);
+ }
+ }
+ }
+
+}
Added:
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/amazons3/AmazonS3Config.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/amazons3/AmazonS3Config.java?rev=1697991&view=auto
==============================================================================
---
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/amazons3/AmazonS3Config.java
(added)
+++
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/amazons3/AmazonS3Config.java
Wed Aug 26 19:03:55 2015
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.manifoldcf.authorities.authorities.amazons3;
+
+import org.apache.manifoldcf.crawler.connectors.BaseRepositoryConnector;
+
+/**
+ * @author Kuhajeyan
+ *
+ */
+public class AmazonS3Config {
+ public static final String AWS_ACCESS_KEY = "aws_access_key";
+
+ public static final String AWS_SECRET_KEY = "aws_secret_key";
+
+ public static final String AMAZONS3_HOST = "amazons3_host";
+
+ public static final String AMAZONS3_PORT = "amazons3_port";
+
+ public static final String AMAZONS3_PROTOCOL = "amazons3_protocol";
+
+ public static final String AMAZONS3_PROXY_HOST = "amazons3_proxy_host";
+
+ public static final String AMAZONS3_PROXY_PORT = "amazons3_proxy_port";
+
+ public static final String AMAZONS3_PROXY_DOMAIN =
"amazons3_proxy_domain";
+
+ public static final String AMAZONS3_PROXY_USERNAME =
"amazons3_proxy_username";
+
+ public static final String AMAZONS3_PROXY_PASSWORD =
"amazons3_proxy_password";
+
+ public static final String AMAZONS3_HOST_DEFAULT = "";
+
+ public static final String AMAZONS3_PORT_DEFAULT = "";
+
+ public static final String AMAZONS3_PROTOCOL_DEFAULT = "http";
+
+ public static final String AMAZONS3_AWS_ACCESS_KEY_DEFAULT = "";
+
+ public static final String AMAZONS3_AWS_SECRET_KEY_DEFAULT = "";
+
+ public static final String AMAZONS3_PROXY_HOST_DEFAULT = "";
+
+ public static final String AMAZONS3_PROXY_PORT_DEFAULT = "";
+
+ public static final String AMAZONS3_PROXY_DOMAIN_DEFAULT = "";
+
+ public static final String AMAZONS3_PROXY_USERNAME_DEFAULT = "";
+
+ public static final String AMAZONS3_PROXY_PASSWORD_DEFAULT = "";
+
+ public static final String AMAZONS3_BUCKETS_DEFAULT = "";
+
+ // Configuration tabs
+ public static final String AMAZONS3_SERVER_TAB_PROPERTY =
"Amazons3AuthorityConnector.Server";
+
+ public static final String AMAZONS3_PROXY_TAB_PROPERTY =
"Amazons3AuthorityConnector.Proxy";
+
+ // Specification tabs
+ public static final String AMAZONS3_BUCKETS_TAB_PROPERTY =
"Amazons3AuthorityConnector.Amazons3Buckets";
+
+ public static final String AMAZONS3_SECURITY_TAB_PROPERTY =
"Amazons3AuthorityConnector.Amazons3Security";
+
+ // Template names for configuration
+ /**
+ * Forward to the javascript to check the configuration parameters
+ */
+ public static final String EDIT_CONFIG_HEADER_FORWARD =
"editConfiguration_amazons3.js";
+
+ public static final String EDIT_CONFIG_FORWARD_SERVER =
"editConfiguration_amazons3_server.html";
+
+ public static final String EDIT_CONFIG_FORWARD_PROXY =
"editConfiguration_amazons3_proxy.html";
+
+ public static final String VIEW_CONFIG_FORWARD =
"viewConfiguration_amazons3.html";
+
+
+
+
+
+
+ //////
+ public static final int CHARACTER_LIMIT = 1000000;
+
+ public static final String DOCUMENT_URI_FORMAT =
"%s.s3.amazonaws.com/%s";
+
+
+
+ /** Deny access token for default authority */
+ public final static String defaultAuthorityDenyToken =
BaseRepositoryConnector.GLOBAL_DENY_TOKEN;
+
+ public static final String JOB_STARTPOINT_NODE_TYPE = "startpoint";
+
+ public static final String JOB_BUCKETS_ATTRIBUTE = "s3buckets";
+}
Added:
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/amazons3/Messages.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/amazons3/Messages.java?rev=1697991&view=auto
==============================================================================
---
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/amazons3/Messages.java
(added)
+++
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/amazons3/Messages.java
Wed Aug 26 19:03:55 2015
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.manifoldcf.authorities.authorities.amazons3;
+
+import java.util.Locale;
+import java.util.Map;
+
+import org.apache.manifoldcf.core.interfaces.IHTTPOutput;
+import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+
+/**
+ * @author Kuhajeyan
+ *
+ */
+public class Messages extends org.apache.manifoldcf.ui.i18n.Messages
+{
+ public static final String
DEFAULT_BUNDLE_NAME="org.apache.manifoldcf.authorities.authorities.amazons3.common";
+ public static final String
DEFAULT_PATH_NAME="org.apache.manifoldcf.authorities.authorities.amazons3";
+
+ /** Constructor - do no instantiate
+ */
+ protected Messages()
+ {
+ }
+
+ public static String getString(Locale locale, String messageKey)
+ {
+ return getString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+ }
+
+ public static String getAttributeString(Locale locale, String messageKey)
+ {
+ return getAttributeString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+ }
+
+ public static String getBodyString(Locale locale, String messageKey)
+ {
+ return getBodyString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+ }
+
+ public static String getAttributeJavascriptString(Locale locale, String
messageKey)
+ {
+ return getAttributeJavascriptString(DEFAULT_BUNDLE_NAME, locale,
messageKey, null);
+ }
+
+ public static String getBodyJavascriptString(Locale locale, String
messageKey)
+ {
+ return getBodyJavascriptString(DEFAULT_BUNDLE_NAME, locale, messageKey,
null);
+ }
+
+ public static String getString(Locale locale, String messageKey, Object[]
args)
+ {
+ return getString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+ }
+
+ public static String getAttributeString(Locale locale, String messageKey,
Object[] args)
+ {
+ return getAttributeString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+ }
+
+ public static String getBodyString(Locale locale, String messageKey,
Object[] args)
+ {
+ return getBodyString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+ }
+
+ public static String getAttributeJavascriptString(Locale locale, String
messageKey, Object[] args)
+ {
+ return getAttributeJavascriptString(DEFAULT_BUNDLE_NAME, locale,
messageKey, args);
+ }
+
+ public static String getBodyJavascriptString(Locale locale, String
messageKey, Object[] args)
+ {
+ return getBodyJavascriptString(DEFAULT_BUNDLE_NAME, locale, messageKey,
args);
+ }
+
+ // More general methods which allow bundlenames and class loaders to be
specified.
+
+ public static String getString(String bundleName, Locale locale, String
messageKey, Object[] args)
+ {
+ return getString(Messages.class, bundleName, locale, messageKey, args);
+ }
+
+ public static String getAttributeString(String bundleName, Locale locale,
String messageKey, Object[] args)
+ {
+ return getAttributeString(Messages.class, bundleName, locale, messageKey,
args);
+ }
+
+ public static String getBodyString(String bundleName, Locale locale, String
messageKey, Object[] args)
+ {
+ return getBodyString(Messages.class, bundleName, locale, messageKey, args);
+ }
+
+ public static String getAttributeJavascriptString(String bundleName, Locale
locale, String messageKey, Object[] args)
+ {
+ return getAttributeJavascriptString(Messages.class, bundleName, locale,
messageKey, args);
+ }
+
+ public static String getBodyJavascriptString(String bundleName, Locale
locale, String messageKey, Object[] args)
+ {
+ return getBodyJavascriptString(Messages.class, bundleName, locale,
messageKey, args);
+ }
+
+ // Resource output
+
+ public static void outputResource(IHTTPOutput output, Locale locale, String
resourceKey,
+ Map<String,String> substitutionParameters, boolean mapToUpperCase)
+ throws ManifoldCFException
+ {
+ outputResource(output,Messages.class,DEFAULT_PATH_NAME,locale,resourceKey,
+ substitutionParameters,mapToUpperCase);
+ }
+
+ public static void outputResourceWithVelocity(IHTTPOutput output, Locale
locale, String resourceKey,
+ Map<String,String> substitutionParameters, boolean mapToUpperCase)
+ throws ManifoldCFException
+ {
+
outputResourceWithVelocity(output,Messages.class,DEFAULT_BUNDLE_NAME,DEFAULT_PATH_NAME,locale,resourceKey,
+ substitutionParameters,mapToUpperCase);
+ }
+
+ public static void outputResourceWithVelocity(IHTTPOutput output, Locale
locale, String resourceKey,
+ Map<String,Object> contextObjects)
+ throws ManifoldCFException
+ {
+
outputResourceWithVelocity(output,Messages.class,DEFAULT_BUNDLE_NAME,DEFAULT_PATH_NAME,locale,resourceKey,
+ contextObjects);
+ }
+
+}
+
Added:
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/amazons3/ResponseException.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/amazons3/ResponseException.java?rev=1697991&view=auto
==============================================================================
---
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/amazons3/ResponseException.java
(added)
+++
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/amazons3/ResponseException.java
Wed Aug 26 19:03:55 2015
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+* http://www.apache.org/licenses/LICENSE-2.0
+ *
+* Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.manifoldcf.authorities.authorities.amazons3;
+
+/**
+ * @author Kuhajeyan
+ *
+ */
+public class ResponseException extends Exception {
+
+ public ResponseException(String msg) {
+ super(msg);
+ }
+
+ public ResponseException(String msg, Throwable cause) {
+ super(msg, cause);
+ }
+}
Added:
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/connectors/common/amazons3/S3Artifact.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/connectors/common/amazons3/S3Artifact.java?rev=1697991&view=auto
==============================================================================
---
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/connectors/common/amazons3/S3Artifact.java
(added)
+++
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/connectors/common/amazons3/S3Artifact.java
Wed Aug 26 19:03:55 2015
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+* http://www.apache.org/licenses/LICENSE-2.0
+ *
+* Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.manifoldcf.connectors.common.amazons3;
+
+/**
+ *
+ * @author Kuhajeyan
+ *
+ */
+public class S3Artifact {
+
+ private String bucketName;
+
+ private String key;
+
+ public S3Artifact() {
+
+ }
+
+ public S3Artifact(String bucketName, String key) {
+
+ this.bucketName = bucketName;
+ this.key = key;
+ }
+
+ public String getBucketName() {
+ return bucketName;
+ }
+
+ public void setBucketName(String bucketName) {
+ this.bucketName = bucketName;
+ }
+
+ public String getKey() {
+ return key;
+ }
+
+ public void setKey(String key) {
+ this.key = key;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if(obj == null)
+ return false;
+
+ if(!(obj instanceof S3Artifact))
+ return false;
+
+ if(obj == this){
+ return true;
+ }
+
+ S3Artifact newObj = (S3Artifact)obj;
+ if(newObj.getBucketName() == this.getBucketName() &&
newObj.getKey() == this.getKey())
+ return true;
+
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return this.getBucketName().hashCode() +
this.getKey().hashCode() + 345;
+ }
+
+}
Added:
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/connectors/common/amazons3/XThreadBuffer.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/connectors/common/amazons3/XThreadBuffer.java?rev=1697991&view=auto
==============================================================================
---
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/connectors/common/amazons3/XThreadBuffer.java
(added)
+++
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/connectors/common/amazons3/XThreadBuffer.java
Wed Aug 26 19:03:55 2015
@@ -0,0 +1,88 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.connectors.common.amazons3;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.manifoldcf.crawler.system.Logging;
+
+/**
+ * Generic XThread class
+ * @author Kuhajeyan
+ *
+ * @param <T>
+ */
+public class XThreadBuffer<T> {
+ protected static int MAX_SIZE = 1024;
+
+ protected List<T> buffer = Collections.synchronizedList(new
ArrayList<T>(
+ MAX_SIZE));
+
+ protected boolean complete = false;
+
+ protected boolean abandoned = false;
+
+ /** Constructor */
+ public XThreadBuffer() {
+ }
+
+ public synchronized void add(T t) throws InterruptedException {
+ while (buffer.size() == MAX_SIZE && !abandoned)
+ wait();
+ if (abandoned)
+ return;
+ buffer.add(t);
+ // Notify threads that are waiting on there being stuff in the
queue
+ notifyAll();
+ }
+
+ public synchronized void abandon() {
+ abandoned = true;
+ // Notify waiting threads
+ notifyAll();
+ }
+
+ public synchronized T fetch() throws InterruptedException {
+
+ while (buffer.size() == 0 && !complete)
+ {
+ if (Logging.connectors != null) {
+ Logging.connectors.info("thread will be put to
wait");
+ }
+ wait();
+ }
+
+ if (buffer.size() == 0)
+ return null;
+ boolean isBufferFull = (buffer.size() == MAX_SIZE);
+ T rval = buffer.remove(buffer.size() - 1);
+ // Notify those threads waiting on buffer being not completely
full to
+ // wake
+ if (isBufferFull)
+ notifyAll();
+ return rval;
+ }
+
+ public synchronized void signalDone() {
+ complete = true;
+ // Notify threads that are waiting for stuff to appear, because
it won't
+ notifyAll();
+ }
+
+}
Added:
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/AmazonS3Config.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/AmazonS3Config.java?rev=1697991&view=auto
==============================================================================
---
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/AmazonS3Config.java
(added)
+++
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/AmazonS3Config.java
Wed Aug 26 19:03:55 2015
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.manifoldcf.crawler.connectors.amazons3;
+
+import org.apache.manifoldcf.crawler.connectors.BaseRepositoryConnector;
+
+/**
+ * @author Kuhajeyan
+ *
+ */
+public class AmazonS3Config {
+ public static final String AWS_ACCESS_KEY = "aws_access_key";
+
+ public static final String AWS_SECRET_KEY = "aws_secret_key";
+
+ public static final String AMAZONS3_HOST = "amazons3_host";
+
+ public static final String AMAZONS3_PORT = "amazons3_port";
+
+ public static final String AMAZONS3_PROTOCOL = "amazons3_protocol";
+
+ public static final String AMAZONS3_PROXY_HOST = "amazons3_proxy_host";
+
+ public static final String AMAZONS3_PROXY_PORT = "amazons3_proxy_port";
+
+ public static final String AMAZONS3_PROXY_DOMAIN =
"amazons3_proxy_domain";
+
+ public static final String AMAZONS3_PROXY_USERNAME =
"amazons3_proxy_username";
+
+ public static final String AMAZONS3_PROXY_PASSWORD =
"amazons3_proxy_password";
+
+ public static final String AMAZONS3_HOST_DEFAULT = "";
+
+ public static final String AMAZONS3_PORT_DEFAULT = "";
+
+ public static final String AMAZONS3_PROTOCOL_DEFAULT = "http";
+
+ public static final String AMAZONS3_AWS_ACCESS_KEY_DEFAULT = "";
+
+ public static final String AMAZONS3_AWS_SECRET_KEY_DEFAULT = "";
+
+ public static final String AMAZONS3_PROXY_HOST_DEFAULT = "";
+
+ public static final String AMAZONS3_PROXY_PORT_DEFAULT = "";
+
+ public static final String AMAZONS3_PROXY_DOMAIN_DEFAULT = "";
+
+ public static final String AMAZONS3_PROXY_USERNAME_DEFAULT = "";
+
+ public static final String AMAZONS3_PROXY_PASSWORD_DEFAULT = "";
+
+ public static final String AMAZONS3_BUCKETS_DEFAULT = "";
+
+ // Configuration tabs
+ public static final String AMAZONS3_SERVER_TAB_PROPERTY =
"Amazons3RepositoryConnector.Server";
+
+ public static final String AMAZONS3_PROXY_TAB_PROPERTY =
"Amazons3RepositoryConnector.Proxy";
+
+ // Specification tabs
+ public static final String AMAZONS3_BUCKETS_TAB_PROPERTY =
"Amazons3RepositoryConnector.Amazons3Buckets";
+
+ public static final String AMAZONS3_SECURITY_TAB_PROPERTY =
"Amazons3RepositoryConnector.Amazons3Security";
+
+ // Template names for configuration
+ /**
+ * Forward to the javascript to check the configuration parameters
+ */
+ public static final String EDIT_CONFIG_HEADER_FORWARD =
"editConfiguration_amazons3.js";
+
+ public static final String EDIT_CONFIG_FORWARD_SERVER =
"editConfiguration_amazons3_server.html";
+
+ public static final String EDIT_CONFIG_FORWARD_PROXY =
"editConfiguration_amazons3_proxy.html";
+
+ public static final String EDIT_SPEC_HEADER_FORWARD =
"editSpecification_amazons3.js";
+
+ public static final String EDIT_SPEC_FORWARD_BUCKETS =
"editSpecification_buckets.html";
+
+ public static final String VIEW_CONFIG_FORWARD =
"viewConfiguration_amazons3.html";
+
+ public static final String VIEW_SPEC_FORWARD =
"viewSpecification_amazons3.html";
+
+ //job
+ protected static final String RELATIONSHIP_RELATED = "related";
+
+
+
+ public static final String JOB_ACCESS_NODE_TYPE = "access";
+
+ public static final String JOB_TOKEN_ATTRIBUTE = "token";
+
+
+
+
+ //////
+ public static final int CHARACTER_LIMIT = 1000000;
+ public static final String DOCUMENT_URI_FORMAT =
"%s.s3.amazonaws.com/%s";
+ public static final String BUCKET_SPLITTER = ",";
+ public static final String STD_SEPARATOR_BUCKET_AND_KEY =
BUCKET_SPLITTER;
+
+
+
+ /** Deny access token for default authority */
+ public final static String defaultAuthorityDenyToken =
BaseRepositoryConnector.GLOBAL_DENY_TOKEN;
+
+ public static final String JOB_STARTPOINT_NODE_TYPE = "startpoint";
+
+ public static final String JOB_BUCKETS_ATTRIBUTE = "s3buckets";
+}
Added:
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/AmazonS3Connector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/AmazonS3Connector.java?rev=1697991&view=auto
==============================================================================
---
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/AmazonS3Connector.java
(added)
+++
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/AmazonS3Connector.java
Wed Aug 26 19:03:55 2015
@@ -0,0 +1,918 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.manifoldcf.crawler.connectors.amazons3;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InterruptedIOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
+import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
+import org.apache.manifoldcf.connectors.common.amazons3.S3Artifact;
+import org.apache.manifoldcf.connectors.common.amazons3.XThreadBuffer;
+import org.apache.manifoldcf.core.connector.BaseConnector;
+import org.apache.manifoldcf.core.interfaces.ConfigParams;
+import org.apache.manifoldcf.core.interfaces.IHTTPOutput;
+import org.apache.manifoldcf.core.interfaces.IPasswordMapperActivity;
+import org.apache.manifoldcf.core.interfaces.IPostParameters;
+import org.apache.manifoldcf.core.interfaces.IThreadContext;
+import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+import org.apache.manifoldcf.core.interfaces.Specification;
+import org.apache.manifoldcf.core.interfaces.SpecificationNode;
+import org.apache.manifoldcf.crawler.connectors.BaseRepositoryConnector;
+import org.apache.manifoldcf.crawler.interfaces.IExistingVersions;
+import org.apache.manifoldcf.crawler.interfaces.IProcessActivity;
+import org.apache.manifoldcf.crawler.interfaces.ISeedingActivity;
+import org.apache.manifoldcf.crawler.system.Logging;
+import org.apache.tika.Tika;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.AutoDetectParser;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.sax.BodyContentHandler;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.AmazonServiceException;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.model.AccessControlList;
+import com.amazonaws.services.s3.model.Bucket;
+import com.amazonaws.services.s3.model.CanonicalGrantee;
+import com.amazonaws.services.s3.model.GetObjectRequest;
+import com.amazonaws.services.s3.model.Grant;
+import com.amazonaws.services.s3.model.Grantee;
+import com.amazonaws.services.s3.model.ListObjectsRequest;
+import com.amazonaws.services.s3.model.ObjectListing;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.Owner;
+import com.amazonaws.services.s3.model.S3Object;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+
+/**
+ * @author Kuhajeyan
+ *
+ */
+public class AmazonS3Connector extends BaseRepositoryConnector {
+
+ private static final String BUCKET_SPLITTER = ",";
+
+ private static final String TAB_NAME = "TabName";
+
+ private static final String SELECTED_NUM = "SelectedNum";
+
+ private static final String SEQ_NUM = "SeqNum";
+
+ protected final static String ACTIVITY_READ = "read document";
+
+ protected long lastSessionFetch = -1L;
+
+ protected static final long timeToRelease = 300000L;
+
+ protected AmazonS3 amazonS3;
+
+ protected boolean connected = false;
+
+ protected String amazons3ProxyHost = null;
+
+ protected String amazons3ProxyPort = null;
+
+ protected String amazons3ProxyDomain = null;
+
+ protected String amazons3ProxyUserName = null;
+
+ protected String amazons3ProxyPassword = null;
+
+ protected String amazons3AwsAccessKey = null;
+
+ protected String amazons3AwsSecretKey = null;
+
+ private static final String STD_SEPARATOR_BUCKET_AND_KEY =
BUCKET_SPLITTER;
+
+ private String[] buckets;
+
+ private DocumentProcess documentProcess;
+
+ public AmazonS3Connector() {
+ super();
+ documentProcess = new GenericDocumentProcess();
+ }
+
+ @Override
+ public String[] getActivitiesList() {
+ return new String[] { ACTIVITY_READ };
+ }
+
+ @Override
+ public String[] getBinNames(String documentIdentifier) {
+ return new String[] { amazons3AwsAccessKey };
+ }
+
+ /**
+ * Close the connection. Call this before discarding the connection.
+ */
+ @Override
+ public void disconnect() throws ManifoldCFException {
+ amazons3AwsAccessKey = null;
+ amazons3AwsSecretKey = null;
+
+ amazons3ProxyHost = null;
+ amazons3ProxyPort = null;
+ amazons3ProxyDomain = null;
+ amazons3ProxyUserName = null;
+ amazons3ProxyPassword = null;
+ }
+
+ /**
+ * Connect method initializes the configparams
+ * */
+ @Override
+ public void connect(ConfigParams configParams) {
+ super.connect(configParams);
+
+ // aws access and secret keys
+ amazons3AwsAccessKey = configParams
+ .getParameter(AmazonS3Config.AWS_ACCESS_KEY);
+ amazons3AwsSecretKey = configParams
+
.getObfuscatedParameter(AmazonS3Config.AWS_SECRET_KEY);
+
+ // proxy values
+ amazons3ProxyHost = configParams
+
.getParameter(AmazonS3Config.AMAZONS3_PROXY_HOST);
+ amazons3ProxyPort = configParams
+
.getParameter(AmazonS3Config.AMAZONS3_PROXY_PORT);
+ amazons3ProxyDomain = configParams
+
.getParameter(AmazonS3Config.AMAZONS3_PROXY_DOMAIN);
+ amazons3ProxyUserName = configParams
+
.getParameter(AmazonS3Config.AMAZONS3_PROXY_USERNAME);
+ amazons3ProxyPassword = configParams
+
.getObfuscatedParameter(AmazonS3Config.AMAZONS3_PROXY_PASSWORD);
+ }
+
+ /**
+ * Get the Amazons3 client, relevant access keys should have been posted
+ * already
+ * @return
+ */
+ protected AmazonS3 getClient() {
+ if (amazonS3 == null) {
+ try {
+ BasicAWSCredentials awsCreds = new
BasicAWSCredentials(
+ amazons3AwsAccessKey,
amazons3AwsSecretKey);
+ amazonS3 = new AmazonS3Client(awsCreds);
+ }
+ catch (Exception e) {
+ Logging.connectors
+ .error("Error while amazon s3
connectionr", e);
+
+ }
+ }
+ lastSessionFetch = System.currentTimeMillis();
+ return amazonS3;
+ }
+
+ /**
+ *
+ */
+ @Override
+ public String check() throws ManifoldCFException {
+ // connect with amazons3 client
+ Logging.connectors.info("Checking connection");
+
+ try {
+ // invokes the check thread
+ CheckThread checkThread = new CheckThread(getClient());
+ checkThread.start();
+ checkThread.join();// should wait for join
+ if (checkThread.getException() != null) {
+ Throwable thr = checkThread.getException();
+ return "Check exception: " + thr.getMessage();
+ }
+ return checkThread.getResult();
+ }
+ catch (InterruptedException ex) {
+ Logging.connectors.error("Error while checking
connection", ex);
+ throw new ManifoldCFException(ex.getMessage(), ex,
+ ManifoldCFException.INTERRUPTED);
+ }
+ }
+
+ @Override
+ public boolean isConnected() {
+ return amazonS3 != null && amazonS3.getS3AccountOwner() != null;
+ }
+
+ @Override
+ public void poll() throws ManifoldCFException {
+ if (lastSessionFetch == -1L) {
+ return;
+ }
+
+ long currentTime = System.currentTimeMillis();
+ if (currentTime >= lastSessionFetch + timeToRelease) {
+ amazonS3 = null;
+ lastSessionFetch = -1L;
+ }
+ }
+
+ @Override
+ public int getMaxDocumentRequest() {
+ return 1;
+ }
+
+ /**
+ * Return the list of relationship types that this connector recognizes.
+ *
+ * @return the list.
+ */
+ @Override
+ public String[] getRelationshipTypes() {
+ return new String[] { AmazonS3Config.RELATIONSHIP_RELATED };
+ }
+
+ private void fillInServerConfigurationMap(Map<String, Object> newMap,
+ IPasswordMapperActivity mapper, ConfigParams
parameters) {
+
+ String amazons3AccessKey = parameters
+ .getParameter(AmazonS3Config.AWS_ACCESS_KEY);
+ String amazons3SecretKey = parameters
+ .getParameter(AmazonS3Config.AWS_SECRET_KEY);
+
+ // default values
+ if (amazons3AccessKey == null)
+ amazons3AccessKey =
AmazonS3Config.AMAZONS3_AWS_ACCESS_KEY_DEFAULT;
+ if (amazons3SecretKey == null)
+ amazons3SecretKey =
AmazonS3Config.AMAZONS3_AWS_SECRET_KEY_DEFAULT;
+ else
+ amazons3SecretKey =
mapper.mapPasswordToKey(amazons3SecretKey);
+
+ // fill the map
+ newMap.put("AMAZONS3_AWS_ACCESS_KEY", amazons3AccessKey);
+ newMap.put("AMAZONS3_AWS_SECRET_KEY", amazons3SecretKey);
+ }
+
+ private void fillInProxyConfigurationMap(Map<String, Object> newMap,
+ IPasswordMapperActivity mapper, ConfigParams
parameters) {
+ String amazons3ProxyHost = parameters
+
.getParameter(AmazonS3Config.AMAZONS3_PROXY_HOST);
+ String amazons3ProxyPort = parameters
+
.getParameter(AmazonS3Config.AMAZONS3_PROXY_PORT);
+ String amazons3ProxyDomain = parameters
+
.getParameter(AmazonS3Config.AMAZONS3_PROXY_DOMAIN);
+ String amazons3ProxyUserName = parameters
+
.getParameter(AmazonS3Config.AMAZONS3_PROXY_USERNAME);
+ String amazons3ProxyPassword = parameters
+
.getObfuscatedParameter(AmazonS3Config.AMAZONS3_PROXY_PASSWORD);
+
+ if (amazons3ProxyHost == null)
+ amazons3ProxyHost =
AmazonS3Config.AMAZONS3_PROXY_HOST_DEFAULT;
+ if (amazons3ProxyPort == null)
+ amazons3ProxyPort =
AmazonS3Config.AMAZONS3_PROXY_PORT_DEFAULT;
+ if (amazons3ProxyDomain == null)
+ amazons3ProxyDomain =
AmazonS3Config.AMAZONS3_PROXY_DOMAIN_DEFAULT;
+ if (amazons3ProxyUserName == null)
+ amazons3ProxyUserName =
AmazonS3Config.AMAZONS3_PROXY_USERNAME_DEFAULT;
+ if (amazons3ProxyPassword == null)
+ amazons3ProxyPassword =
AmazonS3Config.AMAZONS3_PROXY_PASSWORD_DEFAULT;
+ else
+ amazons3ProxyPassword = mapper
+
.mapPasswordToKey(amazons3ProxyPassword);
+
+ // fill the map
+ newMap.put("AMAZONS3_PROXY_HOST", amazons3ProxyHost);
+ newMap.put("AMAZONS3_PROXY_PORT", amazons3ProxyPort);
+ newMap.put("AMAZONS3_PROXY_DOMAIN", amazons3ProxyDomain);
+ newMap.put("AMAZONS3_PROXY_USERNAME", amazons3ProxyUserName);
+ newMap.put("AMAZONS3_PROXY_PWD", amazons3ProxyPassword);
+ }
+
+ @Override
+ public void viewConfiguration(IThreadContext threadContext,
+ IHTTPOutput out, Locale locale, ConfigParams parameters)
+ throws ManifoldCFException, IOException {
+ Map<String, Object> paramMap = new HashMap<String, Object>();
+
+ // Fill in map from each tab
+ fillInServerConfigurationMap(paramMap, out, parameters);
+ fillInProxyConfigurationMap(paramMap, out, parameters);
+
+ Messages.outputResourceWithVelocity(out, locale,
+ AmazonS3Config.VIEW_CONFIG_FORWARD, paramMap);
+ }
+
+ @Override
+ public void outputConfigurationHeader(IThreadContext threadContext,
+ IHTTPOutput out, Locale locale, ConfigParams parameters,
+ List<String> tabsArray) throws ManifoldCFException,
IOException {
+ // Add the Server tab
+ tabsArray.add(Messages.getString(locale,
+ AmazonS3Config.AMAZONS3_SERVER_TAB_PROPERTY));
+ // Add the Proxy tab
+ tabsArray.add(Messages.getString(locale,
+ AmazonS3Config.AMAZONS3_PROXY_TAB_PROPERTY));
+ // Map the parameters
+ Map<String, Object> paramMap = new HashMap<String, Object>();
+
+ // Fill in the parameters from each tab
+ fillInServerConfigurationMap(paramMap, out, parameters);
+ fillInProxyConfigurationMap(paramMap, out, parameters);
+
+ // Output the Javascript - only one Velocity template for all
tabs
+ Messages.outputResourceWithVelocity(out, locale,
+ AmazonS3Config.EDIT_CONFIG_HEADER_FORWARD,
paramMap);
+ }
+
+ @Override
+ public void outputConfigurationBody(IThreadContext threadContext,
+ IHTTPOutput out, Locale locale, ConfigParams parameters,
+ String tabName) throws ManifoldCFException, IOException
{
+ // Call the Velocity templates for each tab
+ Map<String, Object> paramMap = new HashMap<String, Object>();
+ // Set the tab name
+ paramMap.put(TAB_NAME, tabName);
+
+ // Fill in the parameters
+ fillInServerConfigurationMap(paramMap, out, parameters);
+ fillInProxyConfigurationMap(paramMap, out, parameters);
+
+ // Server tab
+ Messages.outputResourceWithVelocity(out, locale,
+ AmazonS3Config.EDIT_CONFIG_FORWARD_SERVER,
paramMap);
+ // Proxy tab
+ Messages.outputResourceWithVelocity(out, locale,
+ AmazonS3Config.EDIT_CONFIG_FORWARD_PROXY,
paramMap);
+ }
+
+ private static void fillInBucketsSpecificationMap(
+ Map<String, Object> newMap, Specification ds) {
+ String s3Buckets = AmazonS3Config.AMAZONS3_BUCKETS_DEFAULT;
+ newMap.put("AMAZONS3BUCKETS", s3Buckets);
+ s3Buckets = getExcludedBuckets(ds);
+ if (s3Buckets != null && !StringUtils.isEmpty(s3Buckets)) {
+ String[] buckets = s3Buckets.split(BUCKET_SPLITTER);
+
+ newMap.put("AMAZONS3BUCKETS", s3Buckets);
+
+ Logging.connectors.info("resolved s3 bucket values : "
+ s3Buckets);
+ }
+ else {
+ Logging.connectors.info("No exclusion buckets
available");
+ }
+ }
+
+ private static String getExcludedBuckets(Specification ds) {
+ String buckets = null;
+ for (int i = 0; i < ds.getChildCount(); i++) {
+ SpecificationNode sn = ds.getChild(i);
+ if
(sn.getType().equals(AmazonS3Config.JOB_STARTPOINT_NODE_TYPE)) {
+ buckets = sn
+
.getAttributeValue(AmazonS3Config.JOB_BUCKETS_ATTRIBUTE);
+ }
+ }
+ return buckets;
+ }
+
+ @Override
+ public String processConfigurationPost(IThreadContext threadContext,
+ IPostParameters variableContext, Locale locale,
+ ConfigParams parameters) throws ManifoldCFException {
+ // server tab
+ String awsAccessKey =
variableContext.getParameter("aws_access_key");
+ if (awsAccessKey != null) {
+ parameters
+
.setParameter(AmazonS3Config.AWS_ACCESS_KEY, awsAccessKey);
+ }
+ String awsSecretKey =
variableContext.getParameter("aws_secret_key");
+ if (awsSecretKey != null) {
+ // set as obfuscated parameter
+
parameters.setObfuscatedParameter(AmazonS3Config.AWS_SECRET_KEY,
+ awsSecretKey);
+ }
+
+ // proxy tab
+ String amazons3ProxyHost = variableContext
+ .getParameter("amazons3_proxy_host");
+ if (amazons3ProxyHost != null) {
+
parameters.setParameter(AmazonS3Config.AMAZONS3_PROXY_HOST,
+ amazons3ProxyHost);
+ }
+ String amazons3ProxyPort = variableContext
+ .getParameter("amazons3_proxy_port");
+ if (amazons3ProxyPort != null) {
+
parameters.setParameter(AmazonS3Config.AMAZONS3_PROXY_PORT,
+ amazons3ProxyPort);
+ }
+ String amazons3ProxyDomain = variableContext
+ .getParameter("amazons3_proxy_domain");
+ if (amazons3ProxyDomain != null) {
+
parameters.setParameter(AmazonS3Config.AMAZONS3_PROXY_DOMAIN,
+ amazons3ProxyDomain);
+ }
+ String amazons3ProxyUserName = variableContext
+ .getParameter("amazons3_proxy_username");
+ if (amazons3ProxyUserName != null) {
+
parameters.setParameter(AmazonS3Config.AMAZONS3_PROXY_USERNAME,
+ amazons3ProxyUserName);
+ }
+ String amazons3ProxyPassword = variableContext
+ .getParameter("amazons3_proxy_pwd");
+ if (amazons3ProxyPassword != null) {
+ // set as obfuscated parameter
+ parameters.setObfuscatedParameter(
+ AmazonS3Config.AMAZONS3_PROXY_USERNAME,
+ amazons3ProxyUserName);
+ }
+
+ return null;
+ }
+
+ @Override
+ public void viewSpecification(IHTTPOutput out, Locale locale,
+ Specification ds, int connectionSequenceNumber)
+ throws ManifoldCFException, IOException {
+ Map<String, Object> paramMap = new HashMap<String, Object>();
+ paramMap.put(SEQ_NUM,
Integer.toString(connectionSequenceNumber));
+ fillInBucketsSpecificationMap(paramMap, ds);
+ Messages.outputResourceWithVelocity(out, locale,
+ AmazonS3Config.VIEW_SPEC_FORWARD, paramMap);
+ }
+
+ /**
+ * Process a specification post. This method is called at the start of
job's
+ * edit or view page, whenever there is a possibility that form data
for a
+ * connection has been posted. Its purpose is to gather form
information and
+ * modify the document specification accordingly. The name of the posted
+ * form is always "editjob". The connector will be connected before this
+ * method can be called.
+ *
+ * @param variableContext contains the post data, including binary
+ * file-upload information.
+ * @param locale is the locale the output is preferred to be in.
+ * @param ds is the current document specification for this job.
+ * @param connectionSequenceNumber is the unique number of this
connection
+ * within the job.
+ * @return null if all is well, or a string error message if there is an
+ * error that should prevent saving of the job (and cause a redirection
to
+ * an error page).
+ */
+ @Override
+ public String processSpecificationPost(IPostParameters variableContext,
+ Locale locale, Specification ds, int
connectionSequenceNumber)
+ throws ManifoldCFException {
+ String seqPrefix = "s" + connectionSequenceNumber + "_";
+ String s3Buckets = variableContext.getParameter(seqPrefix
+ + AmazonS3Config.JOB_BUCKETS_ATTRIBUTE);
+ // strip off buckets
+ if (StringUtils.isNotEmpty(s3Buckets)) {
+ s3Buckets = s3Buckets.replaceAll("\\s+", "");
+ buckets = s3Buckets.split(BUCKET_SPLITTER);
+
+ if (buckets != null) {
+ int i = 0;
+ while (i < ds.getChildCount()) {
+ SpecificationNode oldNode =
ds.getChild(i);
+ if (oldNode.getType().equals(
+
AmazonS3Config.JOB_STARTPOINT_NODE_TYPE)) {
+ ds.removeChild(i);
+ break;
+ }
+ i++;
+ }
+ SpecificationNode node = new SpecificationNode(
+
AmazonS3Config.JOB_STARTPOINT_NODE_TYPE);
+
node.setAttribute(AmazonS3Config.JOB_BUCKETS_ATTRIBUTE,
+ s3Buckets);
+ ds.addChild(ds.getChildCount(), node);
+ }
+
+ }
+
+ String xc = variableContext.getParameter(seqPrefix +
"tokencount");
+ if (xc != null) {
+ // Delete all tokens first
+ int i = 0;
+ while (i < ds.getChildCount()) {
+ SpecificationNode sn = ds.getChild(i);
+ if
(sn.getType().equals(AmazonS3Config.JOB_ACCESS_NODE_TYPE))
+ ds.removeChild(i);
+ else
+ i++;
+ }
+
+ int accessCount = Integer.parseInt(xc);
+ i = 0;
+ while (i < accessCount) {
+ String accessDescription = "_" +
Integer.toString(i);
+ String accessOpName = seqPrefix + "accessop"
+ + accessDescription;
+ xc = variableContext.getParameter(accessOpName);
+ if (xc != null && xc.equals("Delete")) {
+ // Next row
+ i++;
+ continue;
+ }
+ // Get the stuff we need
+ String accessSpec =
variableContext.getParameter(seqPrefix
+ + "spectoken" +
accessDescription);
+ SpecificationNode node = new SpecificationNode(
+
AmazonS3Config.JOB_ACCESS_NODE_TYPE);
+
node.setAttribute(AmazonS3Config.JOB_TOKEN_ATTRIBUTE,
+ accessSpec);
+ ds.addChild(ds.getChildCount(), node);
+ i++;
+ }
+
+ String op = variableContext.getParameter(seqPrefix +
"accessop");
+ if (op != null && op.equals("Add")) {
+ String accessspec =
variableContext.getParameter(seqPrefix
+ + "spectoken");
+ SpecificationNode node = new SpecificationNode(
+
AmazonS3Config.JOB_ACCESS_NODE_TYPE);
+
node.setAttribute(AmazonS3Config.JOB_TOKEN_ATTRIBUTE,
+ accessspec);
+ ds.addChild(ds.getChildCount(), node);
+ }
+ }
+
+ return null;
+ }
+
+ @Override
+ public void outputSpecificationBody(IHTTPOutput out, Locale locale,
+ Specification ds, int connectionSequenceNumber,
+ int actualSequenceNumber, String tabName)
+ throws ManifoldCFException, IOException {
+ Map<String, Object> paramMap = new HashMap<String, Object>();
+ paramMap.put(TAB_NAME, tabName);
+ paramMap.put(SEQ_NUM,
Integer.toString(connectionSequenceNumber));
+ paramMap.put(SELECTED_NUM,
Integer.toString(actualSequenceNumber));
+
+ fillInBucketsSpecificationMap(paramMap, ds);
+ Messages.outputResourceWithVelocity(out, locale,
+ AmazonS3Config.EDIT_SPEC_FORWARD_BUCKETS,
paramMap);
+ }
+
+ @Override
+ public void outputSpecificationHeader(IHTTPOutput out, Locale locale,
+ Specification ds, int connectionSequenceNumber,
+ List<String> tabsArray) throws ManifoldCFException,
IOException {
+ tabsArray.add(Messages.getString(locale,
+ AmazonS3Config.AMAZONS3_BUCKETS_TAB_PROPERTY));
+
+ Map<String, Object> paramMap = new HashMap<String, Object>();
+ paramMap.put(SEQ_NUM,
Integer.toString(connectionSequenceNumber));
+
+ fillInBucketsSpecificationMap(paramMap, ds);
+ Messages.outputResourceWithVelocity(out, locale,
+ AmazonS3Config.EDIT_SPEC_HEADER_FORWARD,
paramMap);
+ }
+
+ @Override
+ public String addSeedDocuments(ISeedingActivity activities,
+ Specification spec, String lastSeedVersion, long
seedTime,
+ int jobMode) throws ManifoldCFException,
ServiceInterruption {
+
+ long startTime;
+ if (lastSeedVersion == null)
+ startTime = 0L;
+ else {
+ // Unpack seed time from seed version string
+ startTime = new Long(lastSeedVersion).longValue();
+ }
+
+ String[] bucketsToRemove = null;
+ String unparsedBuckets = getExcludedBuckets(spec);
+ if (unparsedBuckets != null &&
StringUtils.isNotEmpty(unparsedBuckets))
+ bucketsToRemove =
unparsedBuckets.split(BUCKET_SPLITTER);
+ // get seeds
+ GeedSeeds(activities, bucketsToRemove);
+
+ return new Long(seedTime).toString();
+ }
+
+ private void GeedSeeds(ISeedingActivity activities, String[] buckets)
+ throws ManifoldCFException, ServiceInterruption {
+ GetSeedsThread t = new GetSeedsThread(getClient(), buckets);
+ try {
+ t.start();
+
+ boolean wasInterrupted = false;
+ try {
+ XThreadBuffer<S3Artifact> seedBuffer =
t.getBuffer();
+ // Pick up the paths, and add them to the
activities, before we
+ // join with the child thread.
+ while (true) {
+ // The only kind of exceptions this can
throw are going to
+ // shut the process down.
+ S3Artifact s3Artifact =
seedBuffer.fetch();
+ if (s3Artifact == null) {
+ Logging.connectors.info("No
artifact retured");
+ break;
+ }
+
+ String issueKey =
s3Artifact.getBucketName()
+ +
STD_SEPARATOR_BUCKET_AND_KEY
+ + s3Artifact.getKey();
+
+ Logging.connectors.info("Issue key is :
" + issueKey);
+ activities.addSeedDocument(issueKey);
+
+ }
+ }
+ catch (InterruptedException e) {
+
+ Logging.connectors.error(e);
+
+ wasInterrupted = true;
+ throw e;
+ }
+ catch (ManifoldCFException e) {
+
+ Logging.connectors.error(e);
+
+ if (e.getErrorCode() ==
ManifoldCFException.INTERRUPTED)
+ wasInterrupted = true;
+ throw e;
+ }
+ finally {
+ if (!wasInterrupted)
+ t.finishUp();
+ }
+ }
+ catch (InterruptedException e) {
+
+ Logging.connectors.error(e);
+
+ t.interrupt();
+ throw new ManifoldCFException("Interrupted: " +
e.getMessage(), e,
+ ManifoldCFException.INTERRUPTED);
+ }
+ catch (java.net.SocketTimeoutException e) {
+
+ Logging.connectors.error(e);
+
+ handleIOException(e);
+ }
+ catch (InterruptedIOException e) {
+
+ Logging.connectors.error(e);
+
+ t.interrupt();
+ handleIOException(e);
+ }
+ catch (IOException e) {
+
+ Logging.connectors.error(e);
+
+ handleIOException(e);
+ }
+ catch (ResponseException e) {
+
+ Logging.connectors.error(e);
+
+ handleResponseException(e);
+ }
+ }
+
+ private static void handleIOException(IOException e)
+ throws ManifoldCFException, ServiceInterruption {
+ if (!(e instanceof java.net.SocketTimeoutException)
+ && (e instanceof InterruptedIOException)) {
+ throw new ManifoldCFException("Interrupted: " +
e.getMessage(), e,
+ ManifoldCFException.INTERRUPTED);
+ }
+ Logging.connectors.warn("Amazons3: IO exception: " +
e.getMessage(), e);
+ long currentTime = System.currentTimeMillis();
+ throw new ServiceInterruption("IO exception: " +
e.getMessage(), e,
+ currentTime + 300000L, currentTime + 3 * 60 *
60000L, -1, false);
+ }
+
+ private static void handleResponseException(ResponseException e)
+ throws ManifoldCFException, ServiceInterruption {
+ throw new ManifoldCFException("Unexpected response: " +
e.getMessage(),
+ e);
+ }
+
+ @Override
+ public void processDocuments(String[] documentIdentifiers,
+ IExistingVersions statuses, Specification spec,
+ IProcessActivity activities, int jobMode,
+ boolean usesDefaultAuthority) throws
ManifoldCFException,
+ ServiceInterruption {
+ AmazonS3 amazons3Client = getClient();
+ if (amazons3Client == null)
+ throw new ManifoldCFException(
+ "Amazon client can not connect at the
moment");
+ documentProcess.doPocessDocument(documentIdentifiers, statuses,
spec,
+ activities, jobMode, usesDefaultAuthority,
amazons3Client);
+
+ }
+
+ protected static class GetSeedsThread extends Thread {
+ protected Throwable exception = null;
+
+ protected String[] bucketsToBeRemoved;
+
+ protected AmazonS3 s3client = null;
+
+ protected XThreadBuffer<S3Artifact> seedBuffer;
+
+ public XThreadBuffer<S3Artifact> getBuffer() {
+ return seedBuffer;
+ }
+
+ public void setBuffer(XThreadBuffer<S3Artifact> buffer) {
+ this.seedBuffer = buffer;
+ }
+
+ public GetSeedsThread(AmazonS3 s3, String[] buckets) {
+ super();
+ this.bucketsToBeRemoved = buckets;
+ this.s3client = s3;
+ seedBuffer = new XThreadBuffer<S3Artifact>();
+ setDaemon(true);
+ }
+
+ @Override
+ public void run() {
+ try {
+ // push the keys for all documents
+ processSeeds();
+ }
+ catch (Exception e) {
+
+ Logging.connectors.error(e);
+ this.exception = e;
+ }
+ finally {
+ seedBuffer.signalDone();
+ }
+ }
+
+ private void processSeeds() {
+
+ if (s3client != null) {
+
+ List<Bucket> listBuckets =
s3client.listBuckets();
+ List<String> refinedBuckets = new
ArrayList<String>();
+
+ for (Bucket bucket : listBuckets) {
+ if (bucketsToBeRemoved != null &&
bucketsToBeRemoved.length > 0 && !Arrays.asList(bucketsToBeRemoved).contains(
+ bucket.getName())) {
+
refinedBuckets.add(bucket.getName());
+ }
+ else{
+
refinedBuckets.add(bucket.getName());
+ }
+ }
+
+ for (String bucket : refinedBuckets) {
+ String bucketName = bucket;
+ try {
+ PushSeeds(bucketName);
+ }
+ catch (Exception e) {
+ Logging.connectors.error(e);
+ }
+ }
+
+ }
+ else {
+ Logging.connectors.info("Could not connect
amazon");
+ }
+ }
+
+ private void PushSeeds(String bucketName) {
+ try {
+ ObjectListing objectListing = s3client
+ .listObjects(new
ListObjectsRequest()
+
.withBucketName(bucketName));
+ for (S3ObjectSummary objectSummary :
objectListing
+ .getObjectSummaries()) {
+ try {
+ addSeed(bucketName,
objectSummary);
+ }
+ catch (Exception e) {
+
+ Logging.connectors.error(e);
+
+ }
+ }
+
+ }
+ catch (Exception e) {
+
+ Logging.connectors.error(e);
+ }
+ }
+
+ private void addSeed(String bucketName, S3ObjectSummary
objectSummary)
+ throws InterruptedException {
+ String objectKey = objectSummary.getKey();
+ String combinedKey = bucketName +
STD_SEPARATOR_BUCKET_AND_KEY
+ + objectKey;
+ // push the key
+ seedBuffer.add(new S3Artifact(bucketName, objectKey));
+
+ Logging.connectors
+ .info("Pused a new key(combined) in
seed buffer : "
+ + combinedKey);
+ }
+
+ public void finishUp() throws InterruptedException, IOException,
+ ResponseException {
+ seedBuffer.abandon();
+ join();
+ Throwable thr = exception;
+ if (thr != null) {
+ if (thr instanceof IOException)
+ throw (IOException) thr;
+ else if (thr instanceof ResponseException)
+ throw (ResponseException) thr;
+ else if (thr instanceof RuntimeException)
+ throw (RuntimeException) thr;
+ else if (thr instanceof Error)
+ throw (Error) thr;
+ else
+ throw new RuntimeException("Unhandled
exception of type: "
+ +
thr.getClass().getName(), thr);
+ }
+ }
+
+ }
+
+ protected static class CheckThread extends Thread {
+ protected String result = "Unknown";
+
+ protected AmazonS3 s3 = null;
+
+ protected Throwable exception = null;
+
+ public CheckThread(AmazonS3 s3) {
+ this.s3 = s3;
+ }
+
+ public String getResult() {
+ return result;
+ }
+
+ public Throwable getException() {
+ return exception;
+ }
+
+ @Override
+ public void run() {
+ try {
+ if (s3 != null) {
+ Owner s3AccountOwner =
s3.getS3AccountOwner();
+ if (s3AccountOwner != null) {
+ result =
StringUtils.isNotEmpty(s3AccountOwner
+
.getDisplayName()) ? "Connection OK"
+ : "Connection
Failed";
+ }
+
+ }
+ }
+ catch (AmazonServiceException e) {
+ result = "Connection Failed : " +
e.getMessage();
+ exception = e;
+
+ Logging.connectors.error(e);
+ }
+ catch (AmazonClientException e) {
+ result = "Connection Failed : " +
e.getMessage();
+ exception = e;
+
+ Logging.connectors.error(e);
+ }
+ }
+ }
+
+}
Added:
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/AmazonS3DocumentProcessUtility.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/AmazonS3DocumentProcessUtility.java?rev=1697991&view=auto
==============================================================================
---
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/AmazonS3DocumentProcessUtility.java
(added)
+++
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/AmazonS3DocumentProcessUtility.java
Wed Aug 26 19:03:55 2015
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.manifoldcf.crawler.connectors.amazons3;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
+import org.apache.manifoldcf.connectors.common.amazons3.S3Artifact;
+import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+import org.apache.manifoldcf.crawler.system.Logging;
+import org.apache.tika.metadata.Metadata;
+
+import com.amazonaws.services.s3.model.CanonicalGrantee;
+import com.amazonaws.services.s3.model.Grant;
+import com.amazonaws.services.s3.model.Grantee;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+
+public class AmazonS3DocumentProcessUtility {
+
+ public AmazonS3DocumentProcessUtility() {
+ super();
+ }
+
+ /**
+ * Get users has the the access the to artifact
+ * @param grants available for artifact
+ * @return
+ */
+ protected String[] getUsers(Set<Grant> grants) {
+ Set<String> users = new HashSet<String>();// no duplicates
+ for (Grant grant : grants) {
+ if (grant != null && grant.getGrantee() != null) {
+ Grantee grantee = grant.getGrantee();
+
+ if (grantee instanceof CanonicalGrantee) {
+ users.add(((CanonicalGrantee)
grantee).getDisplayName());
+ }
+ else {
+ users.add(grantee.getIdentifier());
+ }
+ }
+ }
+
+ return users.toArray(new String[users.size()]);
+ }
+
+ /**
+ * Constructs document URI for s3artifact
+ * @param s3Artifact
+ * @return
+ */
+ protected String getDocumentURI(S3Artifact s3Artifact) {
+ return String.format(AmazonS3Config.DOCUMENT_URI_FORMAT,
+ s3Artifact.getBucketName(),
s3Artifact.getKey());
+ }
+
+ /**
+ * Adds available meta data to repository documetn
+ * @param rd repository document
+ * @param metadata2
+ * @throws ManifoldCFException
+ */
+ protected void addAllMetaData(RepositoryDocument rd, Metadata metadata2)
+ throws ManifoldCFException {
+ for (String field : metadata2.names()) {
+ rd.addField(field,
metadata2.get(field));
+ }
+ }
+
+ /**
+ * Get the s3artifact (document) using the document identifier (
bucket,key)
+ * @param documentIdentifier
+ * @return
+ * @throws ManifoldCFException
+ */
+ protected S3Artifact getS3Artifact(String documentIdentifier) throws
ManifoldCFException {
+ String key;
+ String bucketName = documentIdentifier
+
.split(AmazonS3Config.STD_SEPARATOR_BUCKET_AND_KEY)[0];
+ key =
documentIdentifier.split(AmazonS3Config.STD_SEPARATOR_BUCKET_AND_KEY)[1];
+ if (StringUtils.isEmpty(bucketName) || StringUtils.isEmpty(key))
+ throw new ManifoldCFException("bucket or key name is
empty");
+
+ return new S3Artifact(bucketName, key);
+ }
+
+ protected void addRawMetadata(RepositoryDocument rd, ObjectMetadata
objectMetadata) {
+ Map<String, Object> rawMetadata =
objectMetadata.getRawMetadata();
+
+ for (Entry<String, Object> entry : rawMetadata.entrySet()) {
+ try {
+ String value = entry.getValue().toString();
+ String key = entry.getKey();
+ rd.addField(key, value);
+ }
+ catch (ManifoldCFException e) {
+ Logging.connectors.error("Error while adding
metadata",e);
+ }
+ }
+
+ }
+
+}
\ No newline at end of file
Added:
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/DocumentProcess.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/DocumentProcess.java?rev=1697991&view=auto
==============================================================================
---
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/DocumentProcess.java
(added)
+++
manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/DocumentProcess.java
Wed Aug 26 19:03:55 2015
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.manifoldcf.crawler.connectors.amazons3;
+
+import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+import org.apache.manifoldcf.core.interfaces.Specification;
+import org.apache.manifoldcf.crawler.interfaces.IExistingVersions;
+import org.apache.manifoldcf.crawler.interfaces.IProcessActivity;
+
+import com.amazonaws.services.s3.AmazonS3;
+
+public interface DocumentProcess {
+
+
+ void doPocessDocument(String[] documentIdentifiers,
+ IExistingVersions statuses, Specification spec,
+ IProcessActivity activities, int jobMode,
+ boolean usesDefaultAuthority, AmazonS3 amazons3Client)
+ throws ManifoldCFException;
+
+}
\ No newline at end of file
