Added: manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/GenericDocumentProcess.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/GenericDocumentProcess.java?rev=1697991&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/GenericDocumentProcess.java (added) +++ manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/GenericDocumentProcess.java Wed Aug 26 19:03:55 2015 @@ -0,0 +1,275 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.manifoldcf.crawler.connectors.amazons3; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Date; +import java.util.Set; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.manifoldcf.agents.interfaces.RepositoryDocument; +import org.apache.manifoldcf.agents.interfaces.ServiceInterruption; +import org.apache.manifoldcf.connectors.common.amazons3.S3Artifact; +import org.apache.manifoldcf.core.interfaces.ManifoldCFException; +import org.apache.manifoldcf.core.interfaces.Specification; +import org.apache.manifoldcf.crawler.interfaces.IExistingVersions; +import org.apache.manifoldcf.crawler.interfaces.IProcessActivity; +import org.apache.manifoldcf.crawler.system.Logging; + +import com.amazonaws.AmazonClientException; +import com.amazonaws.AmazonServiceException; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.AccessControlList; +import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.Grant; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.S3Object; + +/** + * Generic amazons3 extractor + * @author Kuhajeyan + * + */ +public class GenericDocumentProcess extends AmazonS3DocumentProcessUtility + implements DocumentProcess { + + /** + * Process documents with out any tika extractor + * @param documentIdentifiers + * @param statuses + * @param spec + * @param activities + * @param jobMode + * @param usesDefaultAuthority + * @param amazons3Client + * @throws ManifoldCFException + */ + @Override + public void doPocessDocument(String[] documentIdentifiers, + IExistingVersions statuses, Specification spec, + IProcessActivity activities, int jobMode, + boolean usesDefaultAuthority, AmazonS3 amazons3Client) + throws ManifoldCFException { + if (amazons3Client == null) + throw new ManifoldCFException( + "Amazon client can not connect at the moment"); + + for (String documentIdentifier : documentIdentifiers) { + try { + if (documentIdentifier == null + || StringUtils.isEmpty(documentIdentifier)) { + Logging.connectors + .warn("Document identifier is empty, document will not be processed"); + continue; + } + + String versionString; + String[] aclsToUse; + + if (documentIdentifier + .split(AmazonS3Config.STD_SEPARATOR_BUCKET_AND_KEY) == null + && documentIdentifier.length() < 1) { + continue; + } + + S3Artifact s3Artifact = getS3Artifact(documentIdentifier); + S3Object s3Obj = amazons3Client.getObject(new GetObjectRequest( + s3Artifact.getBucketName(), s3Artifact.getKey())); + + if (s3Obj == null) { + // no such document in the bucket now + // delete document + activities.deleteDocument(documentIdentifier); + continue; + } + + Logging.connectors.info("Content-Type: " + + s3Obj.getObjectMetadata().getContentType()); + ObjectMetadata objectMetadata = s3Obj.getObjectMetadata(); + + Date lastModified = objectMetadata.getLastModified(); + StringBuilder sb = new StringBuilder(); + if (lastModified == null) { + // remove the content + activities.deleteDocument(documentIdentifier); + continue; + } + + aclsToUse = new String[0]; + + AccessControlList objectAcl = amazons3Client.getObjectAcl( + s3Artifact.getBucketName(), s3Artifact.getKey()); + + Set<Grant> grants = objectAcl.getGrants(); + String[] users = getUsers(grants); + + aclsToUse = users; + + // + sb.append(lastModified.toString()); + versionString = sb.toString(); + + Logging.connectors.debug("version string : " + versionString); + + if (versionString.length() > 0 + && !activities.checkDocumentNeedsReindexing( + documentIdentifier, versionString)) { + Logging.connectors + .info("Document need not to be reindexed : " + + documentIdentifier); + continue; + } + + Logging.connectors + .debug("JIRA: Processing document identifier '" + + documentIdentifier + "'"); + + long startTime = System.currentTimeMillis(); + String errorCode = null; + String errorDesc = null; + Long fileSize = null; + + String mimeType = "text/plain";// default + + // tika works starts + InputStream in = null; + ByteArrayOutputStream bao = new ByteArrayOutputStream(); + + String document = null; + + try { + in = s3Obj.getObjectContent(); + IOUtils.copy(in, bao); + long fileLength = bao.size(); + if(fileLength < 1) + { + Logging.connectors.warn("File length 0"); + continue; + } + + String documentURI = getDocumentURI(s3Artifact); + Logging.connectors.debug("document : " + documentURI); + + try { + if (!activities.checkURLIndexable(documentURI)) { + errorCode = activities.EXCLUDED_URL; + errorDesc = "Excluded because of URL ('" + + documentURI + "')"; + activities.noDocument(documentIdentifier, + versionString); + continue; + } + + if (!activities.checkMimeTypeIndexable(mimeType)) { + errorCode = activities.EXCLUDED_MIMETYPE; + errorDesc = "Excluded because of mime type ('" + + mimeType + "')"; + activities.noDocument(documentIdentifier, + versionString); + continue; + } + if (!activities.checkDateIndexable(lastModified)) { + errorCode = activities.EXCLUDED_DATE; + errorDesc = "Excluded because of date (" + + lastModified + ")"; + activities.noDocument(documentIdentifier, + versionString); + continue; + } + + // otherwise process + RepositoryDocument rd = new RepositoryDocument(); + addRawMetadata(rd, objectMetadata); + // Turn into acls and add into + // description + String[] denyAclsToUse; + if (aclsToUse.length > 0) + denyAclsToUse = new String[] { AmazonS3Config.defaultAuthorityDenyToken }; + else + denyAclsToUse = new String[0]; + rd.setSecurity( + RepositoryDocument.SECURITY_TYPE_DOCUMENT, + aclsToUse, denyAclsToUse); + + rd.setMimeType(mimeType); + + if (lastModified != null) + rd.setModifiedDate(lastModified); + + + + + if (!activities.checkLengthIndexable(fileLength)) { + errorCode = activities.EXCLUDED_LENGTH; + errorDesc = "Excluded because of document length (" + + fileLength + ")"; + activities.noDocument(documentIdentifier, + versionString); + continue; + } + + InputStream is = null; + try { + is = new ByteArrayInputStream(bao.toByteArray()); + rd.setBinary(is, fileLength); + activities.ingestDocumentWithException( + documentIdentifier, versionString, + documentURI, rd); + + errorCode = "OK"; + fileSize = new Long(fileLength); + } + finally { + if (is != null) + IOUtils.closeQuietly(is); + } + + } + catch (ServiceInterruption e) { + Logging.connectors + .error("Error while checking if document is indexable", + e); + } + } + catch (IOException e1) { + Logging.connectors.error("Error while copying stream", e1); + } + finally { + //close output stream + IOUtils.closeQuietly(bao); + + //close input stream + if (in != null) + IOUtils.closeQuietly(in); + } + + } + catch (AmazonServiceException e) { + Logging.connectors.error(e); + } + catch (AmazonClientException e) { + Logging.connectors.error(e); + } + } + + } + +}
Added: manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/Messages.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/Messages.java?rev=1697991&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/Messages.java (added) +++ manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/Messages.java Wed Aug 26 19:03:55 2015 @@ -0,0 +1,144 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * +* http://www.apache.org/licenses/LICENSE-2.0 + * +* Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.manifoldcf.crawler.connectors.amazons3; + +import java.util.Locale; +import java.util.Map; + +import org.apache.manifoldcf.core.interfaces.IHTTPOutput; +import org.apache.manifoldcf.core.interfaces.ManifoldCFException; + +/** + * @author Kuhajeyan + * + */ +public class Messages extends org.apache.manifoldcf.ui.i18n.Messages { + + public static final String DEFAULT_BUNDLE_NAME = "org.apache.manifoldcf.crawler.connectors.amazons3.common"; + public static final String DEFAULT_PATH_NAME = "org.apache.manifoldcf.crawler.connectors.amazons3"; + + protected Messages() { + } + + public static String getString(Locale locale, String messageKey) { + return getString(DEFAULT_BUNDLE_NAME, locale, messageKey, null); + } + + public static String getAttributeString(Locale locale, String messageKey) { + return getAttributeString(DEFAULT_BUNDLE_NAME, locale, messageKey, null); + } + + public static String getBodyString(Locale locale, String messageKey) { + return getBodyString(DEFAULT_BUNDLE_NAME, locale, messageKey, null); + } + + public static String getAttributeJavascriptString(Locale locale, + String messageKey) { + return getAttributeJavascriptString(DEFAULT_BUNDLE_NAME, locale, + messageKey, null); + } + + public static String getBodyJavascriptString(Locale locale, + String messageKey) { + return getBodyJavascriptString(DEFAULT_BUNDLE_NAME, locale, messageKey, + null); + } + + public static String getString(Locale locale, String messageKey, + Object[] args) { + return getString(DEFAULT_BUNDLE_NAME, locale, messageKey, args); + } + + public static String getAttributeString(Locale locale, String messageKey, + Object[] args) { + return getAttributeString(DEFAULT_BUNDLE_NAME, locale, messageKey, args); + } + + public static String getBodyString(Locale locale, String messageKey, + Object[] args) { + return getBodyString(DEFAULT_BUNDLE_NAME, locale, messageKey, args); + } + + public static String getAttributeJavascriptString(Locale locale, + String messageKey, Object[] args) { + return getAttributeJavascriptString(DEFAULT_BUNDLE_NAME, locale, + messageKey, args); + } + + public static String getBodyJavascriptString(Locale locale, + String messageKey, Object[] args) { + return getBodyJavascriptString(DEFAULT_BUNDLE_NAME, locale, messageKey, + args); + } + + // More general methods which allow bundlenames and class loaders to be + // specified. + + public static String getString(String bundleName, Locale locale, + String messageKey, Object[] args) { + return getString(Messages.class, bundleName, locale, messageKey, args); + } + + public static String getAttributeString(String bundleName, Locale locale, + String messageKey, Object[] args) { + return getAttributeString(Messages.class, bundleName, locale, + messageKey, args); + } + + public static String getBodyString(String bundleName, Locale locale, + String messageKey, Object[] args) { + return getBodyString(Messages.class, bundleName, locale, messageKey, + args); + } + + public static String getAttributeJavascriptString(String bundleName, + Locale locale, String messageKey, Object[] args) { + return getAttributeJavascriptString(Messages.class, bundleName, locale, + messageKey, args); + } + + public static String getBodyJavascriptString(String bundleName, + Locale locale, String messageKey, Object[] args) { + return getBodyJavascriptString(Messages.class, bundleName, locale, + messageKey, args); + } + + // Resource output + + public static void outputResource(IHTTPOutput output, Locale locale, + String resourceKey, Map<String, String> substitutionParameters, + boolean mapToUpperCase) throws ManifoldCFException { + outputResource(output, Messages.class, DEFAULT_PATH_NAME, locale, + resourceKey, substitutionParameters, mapToUpperCase); + } + + public static void outputResourceWithVelocity(IHTTPOutput output, + Locale locale, String resourceKey, + Map<String, String> substitutionParameters, boolean mapToUpperCase) + throws ManifoldCFException { + outputResourceWithVelocity(output, Messages.class, DEFAULT_BUNDLE_NAME, + DEFAULT_PATH_NAME, locale, resourceKey, substitutionParameters, + mapToUpperCase); + } + + public static void outputResourceWithVelocity(IHTTPOutput output, + Locale locale, String resourceKey, + Map<String, Object> contextObjects) throws ManifoldCFException { + outputResourceWithVelocity(output, Messages.class, DEFAULT_BUNDLE_NAME, + DEFAULT_PATH_NAME, locale, resourceKey, contextObjects); + } +} Added: manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/ResponseException.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/ResponseException.java?rev=1697991&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/ResponseException.java (added) +++ manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/ResponseException.java Wed Aug 26 19:03:55 2015 @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * +* http://www.apache.org/licenses/LICENSE-2.0 + * +* Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.manifoldcf.crawler.connectors.amazons3; + +/** + * @author Kuhajeyan + * + */ +public class ResponseException extends Exception { + + public ResponseException(String msg) { + super(msg); + } + + public ResponseException(String msg, Throwable cause) { + super(msg, cause); + } +} Added: manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/TikaBasedDocumentProcess.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/TikaBasedDocumentProcess.java?rev=1697991&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/TikaBasedDocumentProcess.java (added) +++ manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/amazons3/TikaBasedDocumentProcess.java Wed Aug 26 19:03:55 2015 @@ -0,0 +1,305 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.manifoldcf.crawler.connectors.amazons3; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Date; +import java.util.Set; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.manifoldcf.agents.interfaces.RepositoryDocument; +import org.apache.manifoldcf.connectors.common.amazons3.S3Artifact; +import org.apache.manifoldcf.core.interfaces.ManifoldCFException; +import org.apache.manifoldcf.core.interfaces.Specification; +import org.apache.manifoldcf.crawler.interfaces.IExistingVersions; +import org.apache.manifoldcf.crawler.interfaces.IProcessActivity; +import org.apache.manifoldcf.crawler.system.Logging; +import org.apache.tika.Tika; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.AutoDetectParser; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.sax.BodyContentHandler; + +import com.amazonaws.AmazonClientException; +import com.amazonaws.AmazonServiceException; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.AccessControlList; +import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.Grant; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.S3Object; + +/** + * Tika based document process + * @author Kuhajeyan + * + */ +public class TikaBasedDocumentProcess extends AmazonS3DocumentProcessUtility implements DocumentProcess { + + + + AutoDetectParser parser; + + BodyContentHandler handler; + + Metadata metadata; + + Tika tika; + + ParseContext context; + public TikaBasedDocumentProcess() { + parser = new AutoDetectParser(); + handler = new BodyContentHandler(AmazonS3Config.CHARACTER_LIMIT); + metadata = new Metadata(); + tika = new Tika(); + context = new ParseContext(); + } + + + + /** + * Process documents based on Tika extractor + * @param documentIdentifiers + * @param statuses + * @param spec + * @param activities + * @param jobMode + * @param usesDefaultAuthority + * @param amazons3Client + * @throws ManifoldCFException + */ + @Override + public void doPocessDocument(String[] documentIdentifiers, + IExistingVersions statuses, Specification spec, + IProcessActivity activities, int jobMode, + boolean usesDefaultAuthority, AmazonS3 amazons3Client ) throws ManifoldCFException { + + if (amazons3Client == null) + throw new ManifoldCFException( + "Amazon client can not connect at the moment"); + String[] acls = null; + + // loop documents and process + for (String documentIdentifier : documentIdentifiers) { + try { + if (documentIdentifier != null + && StringUtils.isNotEmpty(documentIdentifier)) { + String versionString; + String[] aclsToUse; + + if (documentIdentifier.split(AmazonS3Config.STD_SEPARATOR_BUCKET_AND_KEY) == null + && documentIdentifier.length() < 1) { + continue; + } + + S3Artifact s3Artifact = getS3Artifact(documentIdentifier); + S3Object s3Obj = amazons3Client + .getObject(new GetObjectRequest(s3Artifact + .getBucketName(), s3Artifact.getKey())); + + if (s3Obj == null) { + // no such document in the bucket now + // delete document + activities.deleteDocument(documentIdentifier); + continue; + } + + Logging.connectors.info("Content-Type: " + + s3Obj.getObjectMetadata().getContentType()); + ObjectMetadata objectMetadata = s3Obj.getObjectMetadata(); + Date lastModified = objectMetadata.getLastModified(); + StringBuilder sb = new StringBuilder(); + if (lastModified == null) { + // remove the content + activities.deleteDocument(documentIdentifier); + continue; + } + + aclsToUse = new String[0]; + + AccessControlList objectAcl = amazons3Client.getObjectAcl( + s3Artifact.getBucketName(), s3Artifact.getKey()); + + Set<Grant> grants = objectAcl.getGrants(); + String[] users = getUsers(grants); + + aclsToUse = users; + + // + sb.append(lastModified.toString()); + versionString = sb.toString(); + + Logging.connectors.debug("version string : " + + versionString); + + if (versionString.length() > 0 + && !activities.checkDocumentNeedsReindexing( + documentIdentifier, versionString)) { + Logging.connectors + .info("Document need not to be reindexed : " + + documentIdentifier); + continue; + } + + //==== + + Logging.connectors + .debug("JIRA: Processing document identifier '" + + documentIdentifier + "'"); + + long startTime = System.currentTimeMillis(); + String errorCode = null; + String errorDesc = null; + Long fileSize = null; + + try { + String mimeType = "text/plain";// default + + // tika works starts + InputStream in = null; + + + String document = null; + try { + in = s3Obj.getObjectContent(); + + parser.parse(in, handler, metadata, context); + mimeType = tika.detect(in); + document = handler.toString(); + if (document == null) + continue; + metadata.set(Metadata.CONTENT_TYPE, mimeType); + } + catch (Exception e) { + Logging.connectors.error( + "Error while parsing tika contents", e); + } + finally { + if (in != null) + IOUtils.closeQuietly(in); + } + + String documentURI = getDocumentURI(s3Artifact); + + Logging.connectors.debug("document : " + documentURI); + + // need some investigation + if (!activities.checkURLIndexable(documentURI)) { + errorCode = activities.EXCLUDED_URL; + errorDesc = "Excluded because of URL ('" + + documentURI + "')"; + activities.noDocument(documentIdentifier, + versionString); + continue; + } + if (!activities.checkMimeTypeIndexable(mimeType)) { + errorCode = activities.EXCLUDED_MIMETYPE; + errorDesc = "Excluded because of mime type ('" + + mimeType + "')"; + activities.noDocument(documentIdentifier, + versionString); + continue; + } + if (!activities.checkDateIndexable(lastModified)) { + errorCode = activities.EXCLUDED_DATE; + errorDesc = "Excluded because of date (" + + lastModified + ")"; + activities.noDocument(documentIdentifier, + versionString); + continue; + } + + // otherwise process + RepositoryDocument rd = new RepositoryDocument(); + // Turn into acls and add into + // description + String[] denyAclsToUse; + if (aclsToUse.length > 0) + denyAclsToUse = new String[] { AmazonS3Config.defaultAuthorityDenyToken }; + else + denyAclsToUse = new String[0]; + rd.setSecurity( + RepositoryDocument.SECURITY_TYPE_DOCUMENT, + aclsToUse, denyAclsToUse); + + rd.setMimeType(mimeType); + + if (lastModified != null) + rd.setModifiedDate(lastModified); + + // set all meta-data fields + addAllMetaData(rd, metadata); + + // get document + + try { + byte[] documentBytes = document + .getBytes(StandardCharsets.UTF_8); + long fileLength = documentBytes.length; + + if (!activities.checkLengthIndexable(fileLength)) { + errorCode = activities.EXCLUDED_LENGTH; + errorDesc = "Excluded because of document length (" + + fileLength + ")"; + activities.noDocument(documentIdentifier, + versionString); + continue; + } + + InputStream is = new ByteArrayInputStream( + documentBytes); + try { + rd.setBinary(is, fileLength); + activities.ingestDocumentWithException( + documentIdentifier, versionString, + documentURI, rd); + + errorCode = "OK"; + fileSize = new Long(fileLength); + } + finally { + if (is != null) + IOUtils.closeQuietly(is); + } + } + catch (Exception e) { + Logging.connectors.error(e); + } + } + catch (Exception e) { + Logging.connectors.error(e); + } + + } + } + catch (AmazonServiceException e) { + Logging.connectors.error(e); + } + catch (AmazonClientException e) { + Logging.connectors.error(e); + } + + } + + } + + +} Added: manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/native2ascii/org/apache/manifoldcf/authorities/authorities/amazons3/common_en_US.properties URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/native2ascii/org/apache/manifoldcf/authorities/authorities/amazons3/common_en_US.properties?rev=1697991&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/native2ascii/org/apache/manifoldcf/authorities/authorities/amazons3/common_en_US.properties (added) +++ manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/native2ascii/org/apache/manifoldcf/authorities/authorities/amazons3/common_en_US.properties Wed Aug 26 19:03:55 2015 @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Amazons3AuthorityConnector.Proxy=Proxy +Amazons3AuthorityConnector.Server=Server +Amazons3AuthorityConnector.Amazons3Security=Security + +Amazons3AuthorityConnector.Amazons3ProxyPortColon=Port: +Amazons3AuthorityConnector.Amazons3ProxyHostColon=Host: +Amazons3AuthorityConnector.Amazons3ProxyDomainColon=Domain: +Amazons3AuthorityConnector.Amazons3ProxyUsernameColon=UserName: +Amazons3AuthorityConnector.Amazons3ProxyPasswordColon=Password: + + +Amazons3AuthorityConnector.Amazons3ProtocolColon=Protocol: +Amazons3AuthorityConnector.Amazons3HostColon=Host: +Amazons3AuthorityConnector.Amazons3PortColon=Port: +Amazons3AuthorityConnector.AwsAccessKeyColon=Access Key: +Amazons3AuthorityConnector.AwsSecretKeyColon=Secret Key: + + +Amazons3AuthorityConnector.Amazons3HostMustNotIncludeSlash=Host should not include slash +Amazons3AuthorityConnector.Amazons3HostMustNotIncludeSlash=Host should not include slash +Amazons3AuthorityConnector.Amazons3PortMustBeAnInteger=Port value should be an integer + +Amazons3AuthorityConnector.Amazons3ProxyPortMustBeAnInteger=Port value should be an integer +Amazons3AuthorityConnector.Amazons3ProxyHostMustNotIncludeSlash=Host should not include slash + +Amazons3AuthorityConnector.Amazons3HostMustNotBeNull=Host value should not be null +Amazons3AuthorityConnector.Amazons3PortMustBeAnInteger=Port value should be an integer +Amazons3AuthorityConnector.Amazons3AccessKeyShouldNotBeNull=Access key should not be null +Amazons3AuthorityConnector.Amazons3SecretKeyShouldNotBeNull=Secret key should not be null + +Amazons3AuthorityConnector.Amazons3Buckets=Exclude Buckets +Amazons3AuthorityConnector.Amazons3BucketsColon=Exclude Buckets: +Amazons3AuthorityConnector.Amazons3BucketsCannotBeNull=Amazon s3 buckets can not be null + + + + Added: manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/amazons3/common_en_US.properties URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/amazons3/common_en_US.properties?rev=1697991&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/amazons3/common_en_US.properties (added) +++ manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/amazons3/common_en_US.properties Wed Aug 26 19:03:55 2015 @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Amazons3RepositoryConnector.Proxy=Proxy +Amazons3RepositoryConnector.Server=Server +Amazons3RepositoryConnector.Amazons3Security=Security + +Amazons3RepositoryConnector.Amazons3ProxyPortColon=Port: +Amazons3RepositoryConnector.Amazons3ProxyHostColon=Host: +Amazons3RepositoryConnector.Amazons3ProxyDomainColon=Domain: +Amazons3RepositoryConnector.Amazons3ProxyUsernameColon=UserName: +Amazons3RepositoryConnector.Amazons3ProxyPasswordColon=Password: + + +Amazons3RepositoryConnector.Amazons3ProtocolColon=Protocol: +Amazons3RepositoryConnector.Amazons3HostColon=Host: +Amazons3RepositoryConnector.Amazons3PortColon=Port: +Amazons3RepositoryConnector.AwsAccessKeyColon=Access Key: +Amazons3RepositoryConnector.AwsSecretKeyColon=Secret Key: + + +Amazons3RepositoryConnector.Amazons3HostMustNotIncludeSlash=Host should not include slash +Amazons3RepositoryConnector.Amazons3HostMustNotIncludeSlash=Host should not include slash +Amazons3RepositoryConnector.Amazons3PortMustBeAnInteger=Port value should be an integer + +Amazons3RepositoryConnector.Amazons3ProxyPortMustBeAnInteger=Port value should be an integer +Amazons3RepositoryConnector.Amazons3ProxyHostMustNotIncludeSlash=Host should not include slash + +Amazons3RepositoryConnector.Amazons3HostMustNotBeNull=Host value should not be null +Amazons3RepositoryConnector.Amazons3PortMustBeAnInteger=Port value should be an integer +Amazons3RepositoryConnector.Amazons3AccessKeyShouldNotBeNull=Access key should not be null +Amazons3RepositoryConnector.Amazons3SecretKeyShouldNotBeNull=Secret key should not be null + +Amazons3RepositoryConnector.Amazons3Buckets=Exclude Buckets +Amazons3RepositoryConnector.Amazons3BucketsColon=Exclude Buckets: +Amazons3RepositoryConnector.Amazons3BucketsCannotBeNull=Amazon s3 buckets can not be null + + + + Added: manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/authorities/authorities/amazons3/editConfiguration_amazons3.js URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/authorities/authorities/amazons3/editConfiguration_amazons3.js?rev=1697991&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/authorities/authorities/amazons3/editConfiguration_amazons3.js (added) +++ manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/authorities/authorities/amazons3/editConfiguration_amazons3.js Wed Aug 26 19:03:55 2015 @@ -0,0 +1,63 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http:// www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<script type="text/javascript"> +<!-- +function checkConfig() +{ + + if (editconnection.amazons3_proxy_port.value != "" && !isInteger(editconnection.amazons3_proxy_port.value)) + { + alert("$Encoder.bodyJavascriptEscape($ResourceBundle.getString('Amazons3AuthorityConnector.Amazons3ProxyPortMustBeAnInteger'))"); + editconnection.amazons3_proxy_port.focus(); + return false; + } + + if (editconnection.amazons3_proxy_host.value != "" && editconnection.amazons3_proxy_host.value.indexOf("/") != -1) + { + alert("$Encoder.bodyJavascriptEscape($ResourceBundle.getString('Amazons3AuthorityConnector.Amazons3ProxyHostMustNotIncludeSlash'))"); + editconnection.amazons3_proxy_host.focus(); + return false; + } + + return true; +} + +function checkConfigForSave() +{ + + + if (editconnection.aws_access_key.value == "") + { + alert("$Encoder.bodyJavascriptEscape($ResourceBundle.getString('Amazons3AuthorityConnector.Amazons3AccessKeyShouldNotBeNull'))"); + SelectTab("$Encoder.bodyJavascriptEscape($ResourceBundle.getString('Amazons3AuthorityConnector.Server'))"); + editconnection.aws_access_key.focus(); + return false; + } + + if (editconnection.aws_secret_key.value == "") + { + alert("$Encoder.bodyJavascriptEscape($ResourceBundle.getString('Amazons3AuthorityConnector.Amazons3SecretKeyShouldNotBeNull'))"); + SelectTab("$Encoder.bodyJavascriptEscape($ResourceBundle.getString('Amazons3AuthorityConnector.Server'))"); + editconnection.aws_secret_key.focus(); + return false; + } + + return true; +} +//--> +</script> \ No newline at end of file Added: manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/authorities/authorities/amazons3/editConfiguration_amazons3_proxy.html URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/authorities/authorities/amazons3/editConfiguration_amazons3_proxy.html?rev=1697991&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/authorities/authorities/amazons3/editConfiguration_amazons3_proxy.html (added) +++ manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/authorities/authorities/amazons3/editConfiguration_amazons3_proxy.html Wed Aug 26 19:03:55 2015 @@ -0,0 +1,78 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +#if($TabName == $ResourceBundle.getString('Amazons3AuthorityConnector.Proxy')) + +<table class="displaytable"> + <tr><td class="separator" colspan="2"><hr/></td></tr> + + <tr> + <td class="description"> + <nobr>$Encoder.bodyEscape($ResourceBundle.getString('Amazons3AuthorityConnector.Amazons3ProxyHostColon'))</nobr> + </td> + <td class="value"> + <input size="32" type="text" id="amazons3_proxy_host" name="amazons3_proxy_host" value="$Encoder.attributeEscape($AMAZONS3_PROXY_HOST)" /> + </td> + </tr> + + <tr> + <td class="description"> + <nobr>$Encoder.bodyEscape($ResourceBundle.getString('Amazons3AuthorityConnector.Amazons3ProxyPortColon'))</nobr> + </td> + <td class="value"> + <input size="5" type="text" id="amazons3_proxy_port" name="amazons3_proxy_port" value="$Encoder.attributeEscape($AMAZONS3_PROXY_PORT)" /> + </td> + </tr> + + <tr><td class="separator" colspan="2"><hr/></td></tr> + + <tr> + <td class="description"> + <nobr>$Encoder.bodyEscape($ResourceBundle.getString('Amazons3AuthorityConnector.Amazons3ProxyDomainColon'))</nobr> + </td> + <td class="value"> + <input size="32" type="text" id="amazons3_proxy_domain" name="amazons3_proxy_domain" value="$Encoder.attributeEscape($AMAZONS3_PROXY_DOMAIN)" /> + </td> + </tr> + + <tr> + <td class="description"> + <nobr>$Encoder.bodyEscape($ResourceBundle.getString('Amazons3AuthorityConnector.Amazons3ProxyUsernameColon'))</nobr> + </td> + <td class="value"> + <input size="16" type="text" id="amazons3_proxy_username" name="amazons3_proxy_username" value="$Encoder.attributeEscape($AMAZONS3_PROXY_USERNAME)" /> + </td> + </tr> + <tr> + <td class="description"> + <nobr>$Encoder.bodyEscape($ResourceBundle.getString('Amazons3AuthorityConnector.Amazons3ProxyPasswordColon'))</nobr> + </td> + <td class="value"> + <input size="16" type="password" id="amazons3_proxy_pwd" name="amazons3_proxy_pwd" value="$Encoder.attributeEscape($AMAZONS3_PROXY_PWD)" /> + </td> + </tr> +</table> + +#else + +<input type="hidden" name="amazons3_proxy_host" value="$Encoder.attributeEscape($AMAZONS3_PROXY_PORT)" /> +<input type="hidden" name="amazons3_proxy_port" value="$Encoder.attributeEscape($AMAZONS3_PROXY_PORT)" /> +<input type="hidden" name="amazons3_proxy_domain" value="$Encoder.attributeEscape($AMAZONS3_PROXY_DOMAIN)" /> +<input type="hidden" name="amazons3_proxy_username" value="$Encoder.attributeEscape($AMAZONS3_PROXY_USERNAME)" /> +<input type="hidden" name="amazons3_proxy_pwd" value="$Encoder.attributeEscape($AMAZONS3_PROXY_PWD)" /> + +#end Added: manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/authorities/authorities/amazons3/editConfiguration_amazons3_server.html URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/authorities/authorities/amazons3/editConfiguration_amazons3_server.html?rev=1697991&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/authorities/authorities/amazons3/editConfiguration_amazons3_server.html (added) +++ manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/authorities/authorities/amazons3/editConfiguration_amazons3_server.html Wed Aug 26 19:03:55 2015 @@ -0,0 +1,53 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +#if($TabName == $ResourceBundle.getString('Amazons3AuthorityConnector.Server')) + +<table class="displaytable"> + <tr> + <td class="separator" colspan="2"><hr /></td> + </tr> + <tr> + <td class="separator" colspan="2"><hr /></td> + </tr> + + <tr> + <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('Amazons3AuthorityConnector.AwsAccessKeyColon'))</nobr> + </td> + <td class="value"><input size="16" type="text" + id="aws_access_key" name="aws_access_key" + value="$Encoder.attributeEscape($AMAZONS3_AWS_ACCESS_KEY)" /></td> + </tr> + <tr> + <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('Amazons3AuthorityConnector.AwsSecretKeyColon'))</nobr> + </td> + <td class="value"><input size="16" type="password" + id="aws_secret_key" name="aws_secret_key" + value="$Encoder.attributeEscape($AMAZONS3_AWS_SECRET_KEY)" /></td> + </tr> +</table> + +#else + + + +<input type="hidden" name="aws_access_key" + value="$Encoder.attributeEscape($AMAZONS3_AWS_ACCESS_KEY)" /> +<input type="hidden" name="aws_secret_key" + value="$Encoder.attributeEscape($AMAZONS3_AWS_SECRET_KEY)" /> + +#end Added: manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/authorities/authorities/amazons3/viewConfiguration_amazons3.html URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/authorities/authorities/amazons3/viewConfiguration_amazons3.html?rev=1697991&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/authorities/authorities/amazons3/viewConfiguration_amazons3.html (added) +++ manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/authorities/authorities/amazons3/viewConfiguration_amazons3.html Wed Aug 26 19:03:55 2015 @@ -0,0 +1,80 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + + + + <table class="displaytable"> + + <tr> + <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('Amazons3AuthorityConnector.AwsAccessKeyColon'))</nobr> + </td> + <td class="value"><nobr>$Encoder.bodyEscape($AMAZONS3_AWS_ACCESS_KEY)</nobr> + </td> + </tr> + + <tr> + <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('Amazons3AuthorityConnector.AwsSecretKeyColon'))</nobr> + </td> + <td class="value"><nobr>********</nobr></td> + </tr> + + <tr> + <td class="separator" colspan="2"><hr /></td> + </tr> + + <tr> + <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('Amazons3AuthorityConnector.Amazons3ProxyHostColon'))</nobr> + </td> + <td class="value"><nobr>$Encoder.bodyEscape($AMAZONS3_PROXY_HOST)</nobr> + </td> + </tr> + + <tr> + <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('Amazons3AuthorityConnector.Amazons3ProxyPortColon'))</nobr> + </td> + <td class="value"><nobr>$Encoder.bodyEscape($AMAZONS3_PROXY_PORT)</nobr> + </td> + </tr> + + <tr> + <td class="separator" colspan="2"><hr /></td> + </tr> + + <tr> + <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('Amazons3AuthorityConnector.Amazons3ProxyDomainColon'))</nobr> + </td> + <td class="value"><nobr>$Encoder.bodyEscape($AMAZONS3_PROXY_DOMAIN)</nobr> + </td> + </tr> + + <tr> + <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('Amazons3AuthorityConnector.Amazons3ProxyUsernameColon'))</nobr> + </td> + <td class="value"><nobr>$Encoder.bodyEscape($AMAZONS3_PROXY_USERNAME)</nobr> + </td> + </tr> + + <tr> + <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('Amazons3AuthorityConnector.Amazons3ProxyPasswordColon'))</nobr> + </td> + <td class="value"><nobr>********</nobr></td> + </tr> + + + +</table> + Added: manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/editConfiguration_amazons3.js URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/editConfiguration_amazons3.js?rev=1697991&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/editConfiguration_amazons3.js (added) +++ manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/editConfiguration_amazons3.js Wed Aug 26 19:03:55 2015 @@ -0,0 +1,63 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http:// www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<script type="text/javascript"> +<!-- +function checkConfig() +{ + + if (editconnection.amazons3_proxy_port.value != "" && !isInteger(editconnection.amazons3_proxy_port.value)) + { + alert("$Encoder.bodyJavascriptEscape($ResourceBundle.getString('Amazons3RepositoryConnector.Amazons3ProxyPortMustBeAnInteger'))"); + editconnection.amazons3_proxy_port.focus(); + return false; + } + + if (editconnection.amazons3_proxy_host.value != "" && editconnection.amazons3_proxy_host.value.indexOf("/") != -1) + { + alert("$Encoder.bodyJavascriptEscape($ResourceBundle.getString('Amazons3RepositoryConnector.Amazons3ProxyHostMustNotIncludeSlash'))"); + editconnection.amazons3_proxy_host.focus(); + return false; + } + + return true; +} + +function checkConfigForSave() +{ + + + if (editconnection.aws_access_key.value == "") + { + alert("$Encoder.bodyJavascriptEscape($ResourceBundle.getString('Amazons3RepositoryConnector.Amazons3AccessKeyShouldNotBeNull'))"); + SelectTab("$Encoder.bodyJavascriptEscape($ResourceBundle.getString('Amazons3RepositoryConnector.Server'))"); + editconnection.aws_access_key.focus(); + return false; + } + + if (editconnection.aws_secret_key.value == "") + { + alert("$Encoder.bodyJavascriptEscape($ResourceBundle.getString('Amazons3RepositoryConnector.Amazons3SecretKeyShouldNotBeNull'))"); + SelectTab("$Encoder.bodyJavascriptEscape($ResourceBundle.getString('Amazons3RepositoryConnector.Server'))"); + editconnection.aws_secret_key.focus(); + return false; + } + + return true; +} +//--> +</script> \ No newline at end of file Added: manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/editConfiguration_amazons3_proxy.html URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/editConfiguration_amazons3_proxy.html?rev=1697991&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/editConfiguration_amazons3_proxy.html (added) +++ manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/editConfiguration_amazons3_proxy.html Wed Aug 26 19:03:55 2015 @@ -0,0 +1,78 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +#if($TabName == $ResourceBundle.getString('Amazons3RepositoryConnector.Proxy')) + +<table class="displaytable"> + <tr><td class="separator" colspan="2"><hr/></td></tr> + + <tr> + <td class="description"> + <nobr>$Encoder.bodyEscape($ResourceBundle.getString('Amazons3RepositoryConnector.Amazons3ProxyHostColon'))</nobr> + </td> + <td class="value"> + <input size="32" type="text" id="amazons3_proxy_host" name="amazons3_proxy_host" value="$Encoder.attributeEscape($AMAZONS3_PROXY_HOST)" /> + </td> + </tr> + + <tr> + <td class="description"> + <nobr>$Encoder.bodyEscape($ResourceBundle.getString('Amazons3RepositoryConnector.Amazons3ProxyPortColon'))</nobr> + </td> + <td class="value"> + <input size="5" type="text" id="amazons3_proxy_port" name="amazons3_proxy_port" value="$Encoder.attributeEscape($AMAZONS3_PROXY_PORT)" /> + </td> + </tr> + + <tr><td class="separator" colspan="2"><hr/></td></tr> + + <tr> + <td class="description"> + <nobr>$Encoder.bodyEscape($ResourceBundle.getString('Amazons3RepositoryConnector.Amazons3ProxyDomainColon'))</nobr> + </td> + <td class="value"> + <input size="32" type="text" id="amazons3_proxy_domain" name="amazons3_proxy_domain" value="$Encoder.attributeEscape($AMAZONS3_PROXY_DOMAIN)" /> + </td> + </tr> + + <tr> + <td class="description"> + <nobr>$Encoder.bodyEscape($ResourceBundle.getString('Amazons3RepositoryConnector.Amazons3ProxyUsernameColon'))</nobr> + </td> + <td class="value"> + <input size="16" type="text" id="amazons3_proxy_username" name="amazons3_proxy_username" value="$Encoder.attributeEscape($AMAZONS3_PROXY_USERNAME)" /> + </td> + </tr> + <tr> + <td class="description"> + <nobr>$Encoder.bodyEscape($ResourceBundle.getString('Amazons3RepositoryConnector.Amazons3ProxyPasswordColon'))</nobr> + </td> + <td class="value"> + <input size="16" type="password" id="amazons3_proxy_pwd" name="amazons3_proxy_pwd" value="$Encoder.attributeEscape($AMAZONS3_PROXY_PWD)" /> + </td> + </tr> +</table> + +#else + +<input type="hidden" name="amazons3_proxy_host" value="$Encoder.attributeEscape($AMAZONS3_PROXY_PORT)" /> +<input type="hidden" name="amazons3_proxy_port" value="$Encoder.attributeEscape($AMAZONS3_PROXY_PORT)" /> +<input type="hidden" name="amazons3_proxy_domain" value="$Encoder.attributeEscape($AMAZONS3_PROXY_DOMAIN)" /> +<input type="hidden" name="amazons3_proxy_username" value="$Encoder.attributeEscape($AMAZONS3_PROXY_USERNAME)" /> +<input type="hidden" name="amazons3_proxy_pwd" value="$Encoder.attributeEscape($AMAZONS3_PROXY_PWD)" /> + +#end Added: manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/editConfiguration_amazons3_server.html URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/editConfiguration_amazons3_server.html?rev=1697991&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/editConfiguration_amazons3_server.html (added) +++ manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/editConfiguration_amazons3_server.html Wed Aug 26 19:03:55 2015 @@ -0,0 +1,58 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +#if($TabName == +$ResourceBundle.getString('Amazons3RepositoryConnector.Server')) + +<table class="displaytable"> + <tr> + <td class="separator" colspan="2"><hr /></td> + </tr> + + + + + <tr> + <td class="separator" colspan="2"><hr /></td> + </tr> + + <tr> + <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('Amazons3RepositoryConnector.AwsAccessKeyColon'))</nobr> + </td> + <td class="value"><input size="16" type="text" + id="aws_access_key" name="aws_access_key" + value="$Encoder.attributeEscape($AMAZONS3_AWS_ACCESS_KEY)" /></td> + </tr> + <tr> + <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('Amazons3RepositoryConnector.AwsSecretKeyColon'))</nobr> + </td> + <td class="value"><input size="16" type="password" + id="aws_secret_key" name="aws_secret_key" + value="$Encoder.attributeEscape($AMAZONS3_AWS_SECRET_KEY)" /></td> + </tr> +</table> + +#else + + + +<input type="hidden" name="aws_access_key" + value="$Encoder.attributeEscape($AMAZONS3_AWS_ACCESS_KEY)" /> +<input type="hidden" name="aws_secret_key" + value="$Encoder.attributeEscape($AMAZONS3_AWS_SECRET_KEY)" /> + +#end Added: manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/editSpecification_amazons3.js URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/editSpecification_amazons3.js?rev=1697991&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/editSpecification_amazons3.js (added) +++ manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/editSpecification_amazons3.js Wed Aug 26 19:03:55 2015 @@ -0,0 +1,24 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<script type="text/javascript"> +<!-- + + + +//--> +</script> \ No newline at end of file Added: manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/editSpecification_buckets.html URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/editSpecification_buckets.html?rev=1697991&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/editSpecification_buckets.html (added) +++ manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/editSpecification_buckets.html Wed Aug 26 19:03:55 2015 @@ -0,0 +1,41 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + + +#if($TabName == $ResourceBundle.getString('Amazons3RepositoryConnector.Amazons3Buckets') && ${SeqNum} == ${SelectedNum}) + +<table class="displaytable"> + <tr><td class="separator" colspan="2"><hr/></td></tr> + <tr> + <td class="description"> + <nobr> + $Encoder.bodyEscape($ResourceBundle.getString('Amazons3RepositoryConnector.Amazons3BucketsColon')) + </nobr> + </td> + <td class="value"> + <nobr> + <input type="text" size="50" name="s${SeqNum}_s3buckets" value="$Encoder.attributeEscape($AMAZONS3BUCKETS)" /> + </nobr> + </td> + </tr> +</table> + +#else + +<input type="hidden" name="s${SeqNum}_s3buckets" value="$Encoder.attributeEscape($AMAZONS3BUCKETS)" /> + +#end Added: manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/viewConfiguration_amazons3.html URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/viewConfiguration_amazons3.html?rev=1697991&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/viewConfiguration_amazons3.html (added) +++ manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/viewConfiguration_amazons3.html Wed Aug 26 19:03:55 2015 @@ -0,0 +1,76 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<table class="displaytable"> + + <tr> + <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('Amazons3RepositoryConnector.AwsAccessKeyColon'))</nobr> + </td> + <td class="value"><nobr>$Encoder.bodyEscape($AMAZONS3_AWS_ACCESS_KEY)</nobr> + </td> + </tr> + + <tr> + <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('Amazons3RepositoryConnector.AwsSecretKeyColon'))</nobr> + </td> + <td class="value"><nobr>********</nobr></td> + </tr> + + <tr> + <td class="separator" colspan="2"><hr /></td> + </tr> + + <tr> + <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('Amazons3RepositoryConnector.Amazons3ProxyHostColon'))</nobr> + </td> + <td class="value"><nobr>$Encoder.bodyEscape($AMAZONS3_PROXY_HOST)</nobr> + </td> + </tr> + + <tr> + <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('Amazons3RepositoryConnector.Amazons3ProxyPortColon'))</nobr> + </td> + <td class="value"><nobr>$Encoder.bodyEscape($AMAZONS3_PROXY_PORT)</nobr> + </td> + </tr> + + <tr> + <td class="separator" colspan="2"><hr /></td> + </tr> + + <tr> + <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('Amazons3RepositoryConnector.Amazons3ProxyDomainColon'))</nobr> + </td> + <td class="value"><nobr>$Encoder.bodyEscape($AMAZONS3_PROXY_DOMAIN)</nobr> + </td> + </tr> + + <tr> + <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('Amazons3RepositoryConnector.Amazons3ProxyUsernameColon'))</nobr> + </td> + <td class="value"><nobr>$Encoder.bodyEscape($AMAZONS3_PROXY_USERNAME)</nobr> + </td> + </tr> + + <tr> + <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('Amazons3RepositoryConnector.Amazons3ProxyPasswordColon'))</nobr> + </td> + <td class="value"><nobr>********</nobr></td> + </tr> + +</table> + Added: manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/viewSpecification_amazons3.html URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/viewSpecification_amazons3.html?rev=1697991&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/viewSpecification_amazons3.html (added) +++ manifoldcf/branches/CONNECTORS-1233/amazons3/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/amazons3/viewSpecification_amazons3.html Wed Aug 26 19:03:55 2015 @@ -0,0 +1,25 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<table class="displaytable"> + <tr> + <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('Amazons3RepositoryConnector.Amazons3BucketsColon'))</nobr> + </td> + <td class="value"><nobr>$Encoder.bodyEscape($AMAZONS3BUCKETS)</nobr> + </td> + </tr> +</table> Added: manifoldcf/branches/CONNECTORS-1233/amazons3/pom.xml URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1233/amazons3/pom.xml?rev=1697991&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1233/amazons3/pom.xml (added) +++ manifoldcf/branches/CONNECTORS-1233/amazons3/pom.xml Wed Aug 26 19:03:55 2015 @@ -0,0 +1,412 @@ +<?xml version="1.0"?> +<project + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"; + xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> + <parent> + <groupId>org.apache.manifoldcf</groupId> + <artifactId>mcf-connectors</artifactId> + <version>2.1</version> + </parent> + <modelVersion>4.0.0</modelVersion> + + <packaging>jar</packaging> + <artifactId>mcf-amazons3-connector</artifactId> + <name>mcf-amazons3-connector</name> + <groupId>org.apache.manifoldcf</groupId> + + <developers> + <developer> + <name>Gunaratnam Kuhajeyan</name> + <organization></organization> + <organizationUrl></organizationUrl> + <url>kuhajeyan.blogspot.com</url> + </developer> + </developers> + + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> + <project.http.version>1.14.1-beta</project.http.version> + <project.oauth.version>1.14.1-beta</project.oauth.version> + <aws-java-sdk.version>1.9.0</aws-java-sdk.version> + </properties> + + <build> + <defaultGoal>integration-test</defaultGoal> + <sourceDirectory>${basedir}/connector/src/main/java</sourceDirectory> + <testSourceDirectory>${basedir}/connector/src/test/java</testSourceDirectory> + <resources> + <resource> + <directory>${basedir}/connector/src/main/native2ascii</directory> + <includes> + <include>**/*.properties</include> + </includes> + </resource> + <resource> + <directory>${basedir}/connector/src/main/resources</directory> + <includes> + <include>**/*.html</include> + <include>**/*.js</include> + </includes> + </resource> + </resources> + <testResources> + <testResource> + <directory>${basedir}/connector/src/test/resources</directory> + </testResource> + </testResources> + + <plugins> + + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>native2ascii-maven-plugin</artifactId> + <version>1.0-beta-1</version> + <configuration> + <workDir>target/classes</workDir> + </configuration> + <executions> + <execution> + <id>native2ascii-utf8</id> + <goals> + <goal>native2ascii</goal> + </goals> + <configuration> + <encoding>UTF8</encoding> + <includes> + <include>**/*.properties</include> + </includes> + </configuration> + </execution> + </executions> + </plugin> + + <!-- Test plugin configuration --> + <plugin> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <id>copy-war</id> + <phase>generate-resources</phase> + <goals> + <goal>copy</goal> + </goals> + <configuration> + <outputDirectory>target/dependency</outputDirectory> + <artifactItems> + <artifactItem> + <groupId>${project.groupId}</groupId> + <artifactId>mcf-api-service</artifactId> + <version>${project.version}</version> + <type>war</type> + <overWrite>false</overWrite> + <destFileName>mcf-api-service.war</destFileName> + </artifactItem> + <artifactItem> + <groupId>${project.groupId}</groupId> + <artifactId>mcf-authority-service</artifactId> + <version>${project.version}</version> + <type>war</type> + <overWrite>false</overWrite> + <destFileName>mcf-authority-service.war</destFileName> + </artifactItem> + <artifactItem> + <groupId>${project.groupId}</groupId> + <artifactId>mcf-crawler-ui</artifactId> + <version>${project.version}</version> + <type>war</type> + <overWrite>false</overWrite> + <destFileName>mcf-crawler-ui.war</destFileName> + </artifactItem> + </artifactItems> + </configuration> + </execution> + </executions> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <excludes> + <exclude>**/*Postgresql*.java</exclude> + <exclude>**/*MySQL*.java</exclude> + </excludes> + <forkMode>always</forkMode> + <workingDirectory>target/test-output</workingDirectory> + </configuration> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-failsafe-plugin</artifactId> + <version>2.12.3</version> + <configuration> + <skipTests>${skipITs}</skipTests> + <systemPropertyVariables> + <crawlerWarPath>../dependency/mcf-crawler-ui.war</crawlerWarPath> + <authorityserviceWarPath>../dependency/mcf-authority-service.war</authorityserviceWarPath> + <apiWarPath>../dependency/mcf-api-service.war</apiWarPath> + </systemPropertyVariables> + <excludes> + <exclude>**/*Postgresql*.java</exclude> + <exclude>**/*MySQL*.java</exclude> + </excludes> + <forkMode>always</forkMode> + <workingDirectory>target/test-output</workingDirectory> + </configuration> + <executions> + <execution> + <id>integration-test</id> + <goals> + <goal>integration-test</goal> + </goals> + </execution> + <execution> + <id>verify</id> + <goals> + <goal>verify</goal> + </goals> + </execution> + </executions> + </plugin> + + </plugins> + </build> + + <dependencies> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>mcf-core</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>mcf-connector-common</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>mcf-pull-agent</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>mcf-agents</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>mcf-ui-core</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + <version>${commons-lang.version}</version> + <type>jar</type> + </dependency> + + <!-- Testing dependencies --> + + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>${junit.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>mcf-core</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>mcf-agents</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>mcf-pull-agent</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>postgresql</groupId> + <artifactId>postgresql</artifactId> + <version>${postgresql.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.hsqldb</groupId> + <artifactId>hsqldb</artifactId> + <version>${hsqldb.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>mysql</groupId> + <artifactId>mysql-connector-java</artifactId> + <version>${mysql.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>mcf-api-service</artifactId> + <version>${project.version}</version> + <type>war</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>mcf-authority-service</artifactId> + <version>${project.version}</version> + <type>war</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>mcf-crawler-ui</artifactId> + <version>${project.version}</version> + <type>war</type> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-server</artifactId> + <version>${jetty.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-util</artifactId> + <version>${jetty.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-webapp</artifactId> + <version>${jetty.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-servlet</artifactId> + <version>${jetty.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-http</artifactId> + <version>${jetty.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-io</artifactId> + <version>${jetty.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-security</artifactId> + <version>${jetty.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-continuation</artifactId> + <version>${jetty.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-xml</artifactId> + <version>${jetty.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.mortbay.jetty</groupId> + <artifactId>jsp-api-2.1-glassfish</artifactId> + <version>${glassfish.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.mortbay.jetty</groupId> + <artifactId>jsp-2.1-glassfish</artifactId> + <version>${glassfish.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + <version>${slf4j.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-simple</artifactId> + <version>${slf4j.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + <version>1.1.1</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + <version>1.2.16</version> + <scope>provided</scope> + <type>jar</type> + </dependency> + <dependency> + <groupId>com.googlecode.json-simple</groupId> + <artifactId>json-simple</artifactId> + <version>1.1</version> + </dependency> + <dependency> + <groupId>commons-codec</groupId> + <artifactId>commons-codec</artifactId> + <version>1.8</version> + </dependency> + <dependency> + <groupId>com.amazonaws</groupId> + <artifactId>aws-java-sdk-s3</artifactId> + <version>${aws-java-sdk.version}</version> + </dependency> + <dependency> + <groupId>com.amazonaws</groupId> + <artifactId>aws-java-sdk</artifactId> + <version>${aws-java-sdk.version}</version> + </dependency> + <dependency> + <groupId>org.apache.tika</groupId> + <artifactId>tika-parsers</artifactId> + <version>${tika.version}</version> + </dependency> + <dependency> + <groupId>org.apache.tika</groupId> + <artifactId>tika-core</artifactId> + <version>${tika.version}</version> + </dependency> + <dependency> + <groupId>org.apache.tika</groupId> + <artifactId>tika-java7</artifactId> + <version>${tika.version}</version> + </dependency> + </dependencies> +</project>
