update files for new curator
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/a47b088a Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/a47b088a Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/a47b088a Branch: refs/heads/staging-curator Commit: a47b088a8dcc42677d6596692d84c429e5a607c6 Parents: b1c5560 Author: Tom Barber <[email protected]> Authored: Sat Feb 18 23:39:13 2017 +0000 Committer: Tom Barber <[email protected]> Committed: Sat Feb 18 23:39:13 2017 +0000 ---------------------------------------------------------------------- catalog/pom.xml | 10 +- .../commons/validation/DirectoryValidator.java | 31 + .../commons/validation/ValidationElement.java | 60 ++ .../commons/validation/ValidationOutput.java | 62 ++ core/pom.xml | 43 +- crawler/pom.xml | 10 +- curator2/pom.xml | 182 ++++ .../curation/configuration/Configuration.java | 98 ++ .../oodt/cas/curation/directory/Directory.java | 15 + .../curation/directory/DirectoryListing.java | 98 ++ .../cas/curation/directory/FileDirectory.java | 40 + .../oodt/cas/curation/ingest/IngestBackend.java | 182 ++++ .../cas/curation/ingest/IngestException.java | 18 + .../oodt/cas/curation/ingest/InputStruct.java | 19 + .../oodt/cas/curation/ingest/OutputStruct.java | 18 + .../metadata/FlatDirMetadataHandler.java | 63 ++ .../cas/curation/metadata/MetadataBackend.java | 178 ++++ .../cas/curation/metadata/MetadataHandler.java | 33 + .../cas/curation/rest/DirectoryBackend.java | 111 +++ .../curation/rest/ExceptionResponseHandler.java | 52 + .../oodt/cas/curation/rest/IngestRest.java | 111 +++ .../oodt/cas/curation/rest/MetadataRest.java | 139 +++ .../oodt/cas/curation/rest/UploadBackend.java | 79 ++ .../oodt/cas/curation/rest/ValidationRest.java | 65 ++ .../cas/curation/service/CurationService.java | 171 ++++ .../curation/service/CurationServiceConfig.java | 227 +++++ .../cas/curation/service/DirectoryResource.java | 138 +++ .../cas/curation/service/IngestionResource.java | 352 +++++++ .../cas/curation/service/MetadataResource.java | 970 +++++++++++++++++++ .../cas/curation/service/PolicyResource.java | 310 ++++++ .../cas/curation/service/SystemResource.java | 69 ++ .../curation/servlet/CuratorConfMetKeys.java | 56 ++ .../cas/curation/servlet/CuratorServlet.java | 28 + .../cas/curation/structs/ExtractorConfig.java | 78 ++ .../cas/curation/structs/IngestionTask.java | 168 ++++ .../curation/structs/IngestionTaskStatus.java | 36 + .../curation/util/CurationXmlStructFactory.java | 114 +++ .../oodt/cas/curation/util/DateUtils.java | 59 ++ .../curation/util/ExtractorConfigReader.java | 58 ++ .../curation/util/ExtractorConfigWriter.java | 51 + .../apache/oodt/cas/curation/util/SSOUtils.java | 60 ++ .../curation/validation/ValidationBackend.java | 128 +++ .../validation/ValidationException.java | 21 + .../site/resources/images/basic_extractor.jpg | Bin 0 -> 53280 bytes .../src/site/resources/images/basic_filemgr.jpg | Bin 0 -> 72096 bytes .../src/site/resources/images/basic_ingest.jpg | Bin 0 -> 48213 bytes .../src/site/resources/images/basic_login.jpg | Bin 0 -> 26935 bytes .../src/site/resources/images/basic_page.jpg | Bin 0 -> 26115 bytes .../src/site/resources/images/basic_staging.jpg | Bin 0 -> 27100 bytes .../src/site/resources/images/cas-curator.jpg | Bin 0 -> 27536 bytes .../src/site/resources/images/cas-curator.psd | Bin 0 -> 1003720 bytes .../src/site/resources/media/Bach-SuiteNo2.mp3 | Bin 0 -> 4781080 bytes curator2/src/site/site.xml | 32 + curator2/src/site/xdoc/api/index.xml | 105 ++ curator2/src/site/xdoc/development/maven.xml | 175 ++++ curator2/src/site/xdoc/user/advanced.xml | 56 ++ curator2/src/site/xdoc/user/basic.xml | 690 +++++++++++++ filemgr/pom.xml | 26 +- .../oodt/cas/filemgr/catalog/Catalog.java | 2 +- .../oodt/cas/filemgr/catalog/LuceneCatalog.java | 534 +++++----- .../oodt/cas/filemgr/catalog/SpacerCatalog.java | 312 ++++++ .../filemgr/catalog/SpacerCatalogFactory.java | 62 ++ .../catalog/solr/DefaultProductSerializer.java | 42 +- .../cas/filemgr/catalog/solr/SolrCatalog.java | 46 +- .../cas/filemgr/catalog/solr/SolrClient.java | 92 +- .../datatransfer/LocalDataTransferer.java | 49 +- .../oodt/cas/filemgr/structs/Element.java | 24 +- .../filemgr/system/CommonsXmlRpcTransport.java | 183 ++++ .../filemgr/system/XmlRpcFileManagerClient.java | 66 +- .../oodt/cas/filemgr/tools/SolrIndexer.java | 55 +- .../cas/filemgr/util/XmlRpcStructFactory.java | 2 +- .../oodt/cas/filemgr/util/XmlStructFactory.java | 11 +- .../cli/action/TestLuceneQueryCliAction.java | 8 +- filemgr/src/test/resources/filemgr.properties | 5 +- .../MetadataProvidedMetExtractor.java | 32 + pom.xml | 2 + profile/pom.xml | 13 +- protocol/http/pom.xml | 10 +- resource/pom.xml | 10 +- .../resource/system/CommonsXmlRpcTransport.java | 181 ++++ .../system/XmlRpcResourceManagerClient.java | 79 +- sso/pom.xml | 13 +- .../oodt/security/sso/opensso/SSOProxy.java | 134 ++- workflow/pom.xml | 10 +- 84 files changed, 7375 insertions(+), 497 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/catalog/pom.xml ---------------------------------------------------------------------- diff --git a/catalog/pom.xml b/catalog/pom.xml index a957174..7c79eab 100644 --- a/catalog/pom.xml +++ b/catalog/pom.xml @@ -76,8 +76,14 @@ </exclusions> </dependency> <dependency> - <groupId>commons-httpclient</groupId> - <artifactId>commons-httpclient</artifactId> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + <exclusions> + <exclusion> + <artifactId>commons-logging</artifactId> + <groupId>commons-logging</groupId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>commons-io</groupId> http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/commons/src/main/java/org/apache/oodt/commons/validation/DirectoryValidator.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/org/apache/oodt/commons/validation/DirectoryValidator.java b/commons/src/main/java/org/apache/oodt/commons/validation/DirectoryValidator.java new file mode 100644 index 0000000..f62a520 --- /dev/null +++ b/commons/src/main/java/org/apache/oodt/commons/validation/DirectoryValidator.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oodt.commons.validation; + +import java.io.File; +import java.util.Map; + +/** + * Interface for Directory Validation inside CAS Curator + * + * @author tbarber + */ +public interface DirectoryValidator { + + ValidationOutput validate(File f, Map<String,String> stagingpath); + +} http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/commons/src/main/java/org/apache/oodt/commons/validation/ValidationElement.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/org/apache/oodt/commons/validation/ValidationElement.java b/commons/src/main/java/org/apache/oodt/commons/validation/ValidationElement.java new file mode 100644 index 0000000..3bed81d --- /dev/null +++ b/commons/src/main/java/org/apache/oodt/commons/validation/ValidationElement.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oodt.commons.validation; + +/** + * @author tbarber + */ +public class ValidationElement { + + String field; + boolean valid; + String message; + + public ValidationElement() { + } + + public ValidationElement(String field, boolean valid, String message) { + this.field = field; + this.valid = valid; + this.message = message; + } + + public String getField() { + return field; + } + + public void setField(String field) { + this.field = field; + } + + public boolean isValid() { + return valid; + } + + public void setValid(boolean valid) { + this.valid = valid; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } +} http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/commons/src/main/java/org/apache/oodt/commons/validation/ValidationOutput.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/org/apache/oodt/commons/validation/ValidationOutput.java b/commons/src/main/java/org/apache/oodt/commons/validation/ValidationOutput.java new file mode 100644 index 0000000..92fb3be --- /dev/null +++ b/commons/src/main/java/org/apache/oodt/commons/validation/ValidationOutput.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oodt.commons.validation; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +/** + * Validation Response for serialization back to UI. + * + * @author tbarber + */ +public class ValidationOutput { + + String validationpath; + List<ValidationElement> validationelements = new ArrayList<ValidationElement>(); + boolean valid = true; + + public ValidationOutput() { + } + + public String getPath() { + return validationpath; + } + + public void setPath(String path) { + this.validationpath = path; + } + + public List<ValidationElement> getValidationelements() { + return validationelements; + } + + public String getValidationpath() { + return validationpath; + } + + public void addValidationResult(String field, boolean valid, String message){ + ValidationElement validationElement = new ValidationElement(field, valid, message); + this.validationelements.add(validationElement); + if(!valid){ + this.valid=false; + } + } +} http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/core/pom.xml ---------------------------------------------------------------------- diff --git a/core/pom.xml b/core/pom.xml index ad4c77f..35f3a70 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -70,11 +70,6 @@ the License. <dependencyManagement> <dependencies> <dependency> - <groupId>org.apache.solr</groupId> - <artifactId>solr-solrj</artifactId> - <version>1.3.0</version> - </dependency> - <dependency> <groupId>com.amazonaws</groupId> <artifactId>aws-java-sdk</artifactId> <version>1.7.4</version> @@ -123,14 +118,9 @@ the License. <version>1.2.1</version> </dependency> <dependency> - <groupId>commons-httpclient</groupId> - <artifactId>commons-httpclient</artifactId> - <version>3.0</version> - </dependency> - <dependency> <groupId>commons-io</groupId> <artifactId>commons-io</artifactId> - <version>1.4</version> + <version>2.4</version> </dependency> <dependency> <groupId>commons-lang</groupId> @@ -140,7 +130,7 @@ the License. <dependency> <groupId>commons-logging</groupId> <artifactId>commons-logging</artifactId> - <version>1.0.3</version> + <version>1.2</version> </dependency> <dependency> <groupId>commons-pool</groupId> @@ -156,6 +146,12 @@ the License. <groupId>edu.ucar</groupId> <artifactId>netcdf4</artifactId> <version>4.5.5</version> + <exclusions> + <exclusion> + <groupId>commons-httpclient</groupId> + <artifactId>commons-httpclient</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>edu.ucar</groupId> @@ -281,7 +277,13 @@ the License. <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> - <version>4.2.5</version> + <version>4.5.2</version> + <exclusions> + <exclusion> + <artifactId>commons-logging</artifactId> + <groupId>commons-logging</groupId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.apache.jena</groupId> @@ -406,7 +408,12 @@ the License. <dependency> <groupId>org.apache.solr</groupId> <artifactId>solr-core</artifactId> - <version>1.3.0</version> + <version>6.2.1</version> + </dependency> + <dependency> + <groupId>org.apache.solr</groupId> + <artifactId>solr-solrj</artifactId> + <version>6.2.1</version> </dependency> <dependency> <groupId>org.apache.tika</groupId> @@ -506,14 +513,14 @@ the License. <artifactId>slf4j-api</artifactId> <version>1.7.12</version> </dependency> -<!-- <dependency> + <dependency> <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j</artifactId> + <artifactId>slf4j-log4j12</artifactId> <version>1.7.12</version> - </dependency>--> + </dependency> <dependency> <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> + <artifactId>jcl-over-slf4j</artifactId> <version>1.7.12</version> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/crawler/pom.xml ---------------------------------------------------------------------- diff --git a/crawler/pom.xml b/crawler/pom.xml index 1fa731e..2bde05b 100644 --- a/crawler/pom.xml +++ b/crawler/pom.xml @@ -70,8 +70,14 @@ the License. <artifactId>commons-collections</artifactId> </dependency> <dependency> - <groupId>commons-httpclient</groupId> - <artifactId>commons-httpclient</artifactId> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + <exclusions> + <exclusion> + <artifactId>commons-logging</artifactId> + <groupId>commons-logging</groupId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>commons-io</groupId> http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/pom.xml ---------------------------------------------------------------------- diff --git a/curator2/pom.xml b/curator2/pom.xml new file mode 100644 index 0000000..fa2017e --- /dev/null +++ b/curator2/pom.xml @@ -0,0 +1,182 @@ +<?xml version='1.0' encoding='UTF-8'?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one or more contributor +license agreements. See the NOTICE.txt file distributed with this work for +additional information regarding copyright ownership. The ASF licenses this +file to you under the Apache License, Version 2.0 (the "License"); you may not +use this file except in compliance with the License. You may obtain a copy of +the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations under +the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.oodt</groupId> + <artifactId>oodt-core</artifactId> + <version>0.13-SNAPSHOT</version> + <relativePath>../core/pom.xml</relativePath> + </parent> + <artifactId>cas-curator-engine</artifactId> + <packaging>jar</packaging> + <name>CAS Curation Engine</name> + <description>A web application for managing policy for products and files and metadata that have been ingested via the CAS component.</description> + <build> + <plugins /> + </build> + <profiles> + <profile> + <id>audit</id> + <activation> + <activeByDefault>false</activeByDefault> + </activation> + <build> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>rat-maven-plugin</artifactId> + <configuration> + <excludes> + <exclude>**/src/MIT-License.txt</exclude> + <exclude>**/media/Bach-SuiteNo2.mp3</exclude> + </excludes> + </configuration> + <executions> + <execution> + <phase>verify</phase> + <goals> + <goal>check</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> + <repositories> + <repository> + <id>ibiblio</id> + <name>Ibiblio Repository</name> + <layout>default</layout> + <url>http://mirrors.ibiblio.org/pub/mirrors/maven2</url> + <releases> + <enabled>true</enabled> + <updatePolicy>always</updatePolicy> + <checksumPolicy>warn</checksumPolicy> + </releases> + <snapshots> + <enabled>false</enabled> + </snapshots> + </repository> + <repository> + <id>maven2-repository.dev.java.net</id> + <name>Java.net Repository for Maven</name> + <url>http://download.java.net/maven/2/</url> + <layout>default</layout> + </repository> + </repositories> + + <dependencies> + <dependency> + <groupId>org.yaml</groupId> + <artifactId>snakeyaml</artifactId> + <version>1.17</version> + </dependency> + <dependency> + <groupId>org.apache.oodt</groupId> + <artifactId>oodt-commons</artifactId> + <version>${project.parent.version}</version> + </dependency> + <dependency> + <groupId>commons-fileupload</groupId> + <artifactId>commons-fileupload</artifactId> + <version>1.2.1</version> + </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </dependency> + <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + <version>2.3</version> + </dependency> + <dependency> + <groupId>commons-collections</groupId> + <artifactId>commons-collections</artifactId> + <version>3.2.1</version> + </dependency> + <dependency> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + <version>2.4</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.oodt</groupId> + <artifactId>cas-filemgr</artifactId> + <version>${project.parent.version}</version> + </dependency> + <dependency> + <groupId>org.apache.oodt</groupId> + <artifactId>oodt-sso</artifactId> + <version>${project.parent.version}</version> + </dependency> + <dependency> + <groupId>org.apache.oodt</groupId> + <artifactId>cas-crawler</artifactId> + <version>${project.parent.version}</version> + <exclusions> + <exclusion> + <artifactId>tika</artifactId> + <groupId>org.apache.tika</groupId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.oodt</groupId> + <artifactId>cas-metadata</artifactId> + <version>${project.parent.version}</version> + <type>jar</type> + </dependency> + <dependency> + <groupId>net.sf.json-lib</groupId> + <artifactId>json-lib</artifactId> + <version>2.3</version> + <classifier>jdk15</classifier> + </dependency> + <dependency> + <groupId>org.apache.cxf</groupId> + <artifactId>cxf-rt-frontend-jaxrs</artifactId> + <version>3.1.6</version> + </dependency> + <dependency> + <groupId>org.apache.cxf</groupId> + <artifactId>cxf-rt-transports-http</artifactId> + <version>3.1.6</version> + </dependency> + <dependency> + <groupId>org.apache.cxf</groupId> + <artifactId>cxf-rt-transports-http-jetty</artifactId> + <version>3.1.6</version> + </dependency> + <dependency> + <groupId>org.apache.cxf</groupId> + <artifactId>cxf-rt-management</artifactId> + <version>3.1.6</version> + </dependency> + <dependency> + <groupId>com.google.code.gson</groupId> + <artifactId>gson</artifactId> + <version>2.3.1</version> + </dependency> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/configuration/Configuration.java ---------------------------------------------------------------------- diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/configuration/Configuration.java b/curator2/src/main/java/org/apache/oodt/cas/curation/configuration/Configuration.java new file mode 100644 index 0000000..d828e5d --- /dev/null +++ b/curator2/src/main/java/org/apache/oodt/cas/curation/configuration/Configuration.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oodt.cas.curation.configuration; + +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import javax.servlet.ServletContext; + +import org.apache.oodt.cas.metadata.util.PathUtils; + +/** + * Singleton to hold configuration for the curation + * @author starchmd + */ +public class Configuration { + //Config names + public static final String FILLER_METDATA_KEY = "fill"; + public static final String UPLOAD_AREA_CONFIG = "org.apache.oodt.cas.curator.upload.area"; + public static final String STAGING_AREA_CONFIG = "org.apache.oodt.cas.curator.staging.area"; + public static final String ARCHIVE_AREA_CONFIG = "org.apache.oodt.cas.curator.archive.area"; + public static final String METADATA_AREA_CONFIG = "org.apache.oodt.cas.curator.metadata.area"; + public static final String EXTRACTOR_AREA_CONFIG = "org.apache.oodt.cas.curator.extractor.area"; + public static final String FILEMANAGER_URL_CONFIG = "org.apache.oodt.cas.curator.filemanager.url"; + public static final String FILEMANAGER_PROP_CONFIG = "org.apache.oodt.cas.curator.filemanager.prop"; + public static final String DIRECTORYBACKEND_VALIDATOR = "org.apache.oodt.cas.curator.directory.validator"; + public static final String POLICY_UPLOAD_PATH = "org.apache.oodt.cas.curator.dataDefinition.uploadPath"; + + //Stores the configuration object as a Properties object + private static Properties config = new Properties(); + + /** + * Loads a configuration given a servlet context + * @param context - servlet context + */ + @SuppressWarnings("unchecked") + public static void loadConfiguration(ServletContext context) { + //TODO: make sure all above values are loaded + Enumeration<String> inits = context.getInitParameterNames(); + while(inits.hasMoreElements()) { + String key = inits.nextElement(); + set(key,context.getInitParameter(key)); + } + } + /** + * Accessor to wrap properties object implementation + * @param key - configuration key to access + * @return value of configuration + */ + public static String get(String key) { + String ret = config.getProperty(key); + return ret; + } + /** + * Get key with replacement of env vars + * @param key - key to get + * @return value with [VAR] replaced + */ + public static String getWithReplacement(String key) { + String ret = PathUtils.replaceEnvVariables(get(key)); + return ret; + } + /** + * Modifier for global configuration, private to prevent tampering + * @param key - key to modify + * @param val - value to modify + */ + private static void set(String key,String val) { + config.setProperty(key, val); + } + + public static Map<String,String> getAllProperties() { + Enumeration e = config.propertyNames(); + Map<String,String> generatedproperties = new HashMap<String, String>(); + while (e.hasMoreElements()) { + String key = (String) e.nextElement(); + generatedproperties.put(key, getWithReplacement(key)); + } + return generatedproperties; + } + +} http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/directory/Directory.java ---------------------------------------------------------------------- diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/directory/Directory.java b/curator2/src/main/java/org/apache/oodt/cas/curation/directory/Directory.java new file mode 100644 index 0000000..fd4e3a2 --- /dev/null +++ b/curator2/src/main/java/org/apache/oodt/cas/curation/directory/Directory.java @@ -0,0 +1,15 @@ +package org.apache.oodt.cas.curation.directory; + +/** + * Anything that can be a directory listed by curator. Allows implementation of + * different backends to the staging area (not just file system). + * + * @author starchmd + */ +public interface Directory { + /** + * Perform a listing of this directory. + * @return - directory listing + */ + public DirectoryListing list() throws Exception; +} http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/directory/DirectoryListing.java ---------------------------------------------------------------------- diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/directory/DirectoryListing.java b/curator2/src/main/java/org/apache/oodt/cas/curation/directory/DirectoryListing.java new file mode 100644 index 0000000..3c8e0f1 --- /dev/null +++ b/curator2/src/main/java/org/apache/oodt/cas/curation/directory/DirectoryListing.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oodt.cas.curation.directory; + +import org.apache.oodt.cas.curation.configuration.Configuration; +import org.apache.oodt.commons.validation.DirectoryValidator; +import org.apache.oodt.commons.validation.ValidationOutput; + +import java.io.File; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; + +/** + * A directory listing object + * + * @author starchmd + */ +public class DirectoryListing { + //Types of directory objects + public enum Type { + DIRECTORY, + OBJECT + } + public static final String ROOT_NAME = "Root"; + //Attributes of node + DirectoryListing.Type type; + String name; + String path; + ValidationOutput validation; + + //Children listings (only valid for directory types) + List<DirectoryListing> children = new LinkedList<DirectoryListing>(); + /** + * Get a directory listing + * @param type - type of listing + * @param name - name of object + */ + public DirectoryListing(DirectoryListing.Type type,String name,String path, ValidationOutput validation) { + this.name = name; + this.type = type; + this.path = path; + this.children = (type == DirectoryListing.Type.DIRECTORY) ? new LinkedList<DirectoryListing>() : null; + this.validation = validation; + } + /** + * Create a directory listing + * @param paths - list of file paths + * @param validator + * @return top-level directory listing object + */ + public static DirectoryListing lisingFromFileObjects(Collection<File> paths, File root, + DirectoryValidator validator) { + //Shallow copy and sort + List<File> copy = new LinkedList<File>(paths); + Collections.sort(copy); + //Create a stack to hold directories (implementation details) + LinkedList<DirectoryListing> stack = new LinkedList<DirectoryListing>(); + stack.addLast( (root != null && root.isDirectory()) ? + new DirectoryListing(DirectoryListing.Type.DIRECTORY,root.getPath(),root.getPath(), + validator != null ? validator.validate(root, Configuration.getAllProperties()):null) : + new DirectoryListing(DirectoryListing.Type.DIRECTORY,ROOT_NAME,"", null)); + for (File file : paths) { + if (file.equals(root)) + continue; + //Remove all directories off stack until file starts with last's path + while (!file.getPath().startsWith(stack.peekLast().path)) + stack.removeLast(); + //Get type and name of this file path and create dl object + DirectoryListing.Type type = file.isDirectory() ? DirectoryListing.Type.DIRECTORY : DirectoryListing.Type.OBJECT; + DirectoryListing dl = new DirectoryListing(type,file.getName(),file.getPath(), + validator != null ? validator.validate(file, Configuration.getAllProperties()) : + null); + + //Add to last's children + stack.peekLast().children.add(dl); + if (type == DirectoryListing.Type.DIRECTORY) { + stack.addLast(dl); + } + } + return stack.peekFirst(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/directory/FileDirectory.java ---------------------------------------------------------------------- diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/directory/FileDirectory.java b/curator2/src/main/java/org/apache/oodt/cas/curation/directory/FileDirectory.java new file mode 100644 index 0000000..ce2d3f6 --- /dev/null +++ b/curator2/src/main/java/org/apache/oodt/cas/curation/directory/FileDirectory.java @@ -0,0 +1,40 @@ +package org.apache.oodt.cas.curation.directory; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.TrueFileFilter; +import org.apache.oodt.commons.validation.DirectoryValidator; + +/** + * A file-system based directory for listing into the curator. + * + * @author starchmd + */ +public class FileDirectory implements Directory { + + String directory = null; + private DirectoryValidator validator; + /** + * Build the object around a set directory + * @param directory + */ + public FileDirectory(String directory, DirectoryValidator validator) { + this.directory = directory; + this.validator = validator; + } + /* (non-Javadoc) + * @see org.apache.oodt.cas.curation.directory.Directory#list() + */ + @Override + public DirectoryListing list() throws IOException { + Collection<File> listing = FileUtils.listFilesAndDirs(new File(directory),TrueFileFilter.INSTANCE,TrueFileFilter.INSTANCE); + return DirectoryListing.lisingFromFileObjects(listing,listing.iterator().next(), validator); + } + + public void setValidator(DirectoryValidator validator) { + this.validator = validator; + } +} http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/IngestBackend.java ---------------------------------------------------------------------- diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/IngestBackend.java b/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/IngestBackend.java new file mode 100644 index 0000000..c72603e --- /dev/null +++ b/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/IngestBackend.java @@ -0,0 +1,182 @@ +package org.apache.oodt.cas.curation.ingest; + +import java.io.File; +import java.net.URL; +import java.net.URLDecoder; +import java.net.URLEncoder; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.io.FileUtils; +import org.apache.oodt.cas.curation.configuration.Configuration; +import org.apache.oodt.cas.curation.ingest.InputStruct.InputEntry; +import org.apache.oodt.cas.curation.metadata.FlatDirMetadataHandler; +import org.apache.oodt.cas.curation.rest.IngestRest; +import org.apache.oodt.cas.filemgr.ingest.Ingester; +import org.apache.oodt.cas.filemgr.ingest.StdIngester; +import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException; +import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient; +import org.apache.oodt.cas.metadata.Metadata; + +/** + * Backend for the ingest service + * @author starchmd + */ +public class IngestBackend { + private static final String DATA_TRANSFER_SERVICE = "org.apache.oodt.cas.filemgr.datatransfer.LocalDataTransferFactory"; + private static final String IN_PROGRESS = "IN PROGRESS"; + private static final String DONE = "DONE"; + private static final Logger LOG = Logger.getLogger(IngestRest.class.getName()); + + private ConcurrentHashMap<String,List<InputEntry>> current = new ConcurrentHashMap<String,List<InputEntry>>(); + + private XmlRpcFileManagerClient client = null; + private Ingester ingester = null; + private URL url = null; + /** + * Setup this backend + * @throws IngestException + */ + public IngestBackend() throws IngestException { + try { + this.url = new URL(Configuration.getWithReplacement(Configuration.FILEMANAGER_URL_CONFIG)); + this.ingester = new StdIngester(DATA_TRANSFER_SERVICE); + LOG.log(Level.INFO,"Connecting to File Manager at:"+this.url.toString()); + this.client = new XmlRpcFileManagerClient(this.url); + } catch(Exception e) { + LOG.log(Level.WARNING,"Error: problem constructing backend: "+e); + throw new IngestException("Error: problem setting up ingest backend.",e); + } + } + /** + * Ingest based on input + * @param input - input struct + * @param user - user to isolate requests + */ + public void ingest(InputStruct input,String user) { + if (!current.containsKey(user)) { + current.put(user, new LinkedList<InputEntry>()); + } + for (InputStruct.InputEntry entry : input.entries) { + current.get(user).add(entry); + } + for (InputStruct.InputEntry entry : input.entries) { + try { + ingest(entry.file,user); + } catch(IngestException e) { + entry.error = e; + } + } + } + /** + * Ingests a single file + * @param file - file to ingest + * @param user - user to isolate requests + * @throws IngestException - error on ingestion + */ + private void ingest(String file, String user) throws IngestException { + try { + file = URLDecoder.decode(file); + File full = null; + if (!file.startsWith("/")) { + String parent = new File(Configuration.getWithReplacement(Configuration.STAGING_AREA_CONFIG)).getParent(); + full = new File(parent,file); + } else { + full = new File(file); + } + FlatDirMetadataHandler handler = new FlatDirMetadataHandler(); + Metadata meta = handler.get(file,user); + System.out.println("File: "+file+" URL: "+this.url+" Full: "+full.getAbsoluteFile()+" Metadata: "+meta); + ingester.ingest(this.url, full.getAbsoluteFile(), meta); + //Remove metadata file after successful ingest + handler.remove(file,user); + org.apache.commons.io.FileUtils.deleteQuietly(full); + LOG.log(Level.FINE,"Checking if directory can be removed:"+full.getParent()); + File p = full.getParentFile(); + String test = Configuration.getWithReplacement(Configuration.STAGING_AREA_CONFIG).trim(); + + if(test.endsWith("/")){ + test=test.substring(0, test.length()-1);; + } + while(p!=null && !p.getAbsolutePath().equals(test)) + if(p.exists() && p.isDirectory()){ + + if(p.list().length==0){ + FileUtils.deleteDirectory(p); + } + p = p.getParentFile(); + } + } catch(Exception e) { + LOG.log(Level.WARNING,"Error: failed ingesting product: "+e); + throw new IngestException("Error: problem while ingesting",e); + } + } + /** + * Check the status of the currently ingested items + * @return output struct + */ + public OutputStruct status(String user) throws IngestException { + try { + //List<InputStruct.InputEntry> torm = new LinkedList<InputStruct.InputEntry>(); + List<OutputStruct.OutputEntry> ret = new LinkedList<OutputStruct.OutputEntry>(); + if (this.current.get(user) != null) { + for (InputStruct.InputEntry entry : current.get(user)) { + OutputStruct.OutputEntry temp = new OutputStruct.OutputEntry(); + temp.file = entry.file; + temp.pname = entry.pname; + temp.timestamp = entry.timestamp; + try { + + if (entry.error != null) { + //torm.add(entry); + temp.status = "Error: " + entry.error.getMessage(); + } else if (client.hasProduct(entry.pname)) { + //torm.add(entry); + temp.product = client.getProductByName(entry.pname).getProductId(); + temp.status = DONE; + } else { + temp.status = IN_PROGRESS; + } + ret.add(temp); + } + catch (CatalogException e){ + LOG.log(Level.WARNING,"Error: failed fetching product: "+e.getLocalizedMessage()); + } + } + } + //for (InputStruct.InputEntry entry : torm) { + // this.current.get(user).remove(entry); + //} + //if (this.current.get(user).isEmpty()) { + // this.current.remove(user); + //} + OutputStruct out = new OutputStruct(); + out.status = ret; + return out; + } catch (Exception e) { + LOG.log(Level.WARNING,"Error: failed checking status: "+e); + throw new IngestException("Error: failed to check status",e); + } + } + /** + * Clears any errors registered in this system + */ + public void clearErrors(String user) { + List<InputStruct.InputEntry> torm = new LinkedList<InputStruct.InputEntry>(); + if (this.current.get(user) == null) { + return; + } + for (InputStruct.InputEntry entry : this.current.get(user)) { + torm.add(entry); + } + for (InputStruct.InputEntry entry : torm) { + this.current.get(user).remove(entry); + } + if (this.current.get(user).isEmpty()) { + this.current.remove(user); + } + } +} http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/IngestException.java ---------------------------------------------------------------------- diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/IngestException.java b/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/IngestException.java new file mode 100644 index 0000000..55043f2 --- /dev/null +++ b/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/IngestException.java @@ -0,0 +1,18 @@ +package org.apache.oodt.cas.curation.ingest; + +/** + * A validation backend exception + * + * @author starchmd + */ +public class IngestException extends Exception { + private static final long serialVersionUID = -7714024388845498231L; + /** + * ctor + * @param message - message + * @param cause - cause of error + */ + public IngestException(String message,Exception cause) { + super(message,cause); + } +} http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/InputStruct.java ---------------------------------------------------------------------- diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/InputStruct.java b/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/InputStruct.java new file mode 100644 index 0000000..42644b2 --- /dev/null +++ b/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/InputStruct.java @@ -0,0 +1,19 @@ +package org.apache.oodt.cas.curation.ingest; + +import java.util.List; + +/** + * Class to represent input for ingest + * @author starchmd + */ +public class InputStruct { + //public String id; + public List<InputStruct.InputEntry> entries; + public static class InputEntry { + public String pname; + public String file; + public long size; + public long timestamp; + public Exception error = null; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/OutputStruct.java ---------------------------------------------------------------------- diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/OutputStruct.java b/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/OutputStruct.java new file mode 100644 index 0000000..e8c6543 --- /dev/null +++ b/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/OutputStruct.java @@ -0,0 +1,18 @@ +package org.apache.oodt.cas.curation.ingest; + +import java.util.List; + +/** + * Class representing output + * @author selina + */ +public class OutputStruct { + public List<OutputStruct.OutputEntry> status; + public static class OutputEntry { + public String file; + public String product; + public String pname; + public String status; + public long timestamp; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/metadata/FlatDirMetadataHandler.java ---------------------------------------------------------------------- diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/metadata/FlatDirMetadataHandler.java b/curator2/src/main/java/org/apache/oodt/cas/curation/metadata/FlatDirMetadataHandler.java new file mode 100644 index 0000000..c0cb9f4 --- /dev/null +++ b/curator2/src/main/java/org/apache/oodt/cas/curation/metadata/FlatDirMetadataHandler.java @@ -0,0 +1,63 @@ +/** + * + */ +package org.apache.oodt.cas.curation.metadata; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; + +import org.apache.oodt.cas.curation.configuration.Configuration; +import org.apache.oodt.cas.metadata.Metadata; +import org.apache.oodt.cas.metadata.SerializableMetadata; + +/** + * A metadata handler that uses a directory and fills it with flat-files. + * @author starchmd + */ +public class FlatDirMetadataHandler implements MetadataHandler { + public static final String ENCODING = "UTF-8"; + + /* (non-Javadoc) + * @see org.apache.oodt.cas.curation.metadata.MetadataHandler#get(java.lang.String) + */ + @Override + public Metadata get(String file,String user) throws InstantiationException, FileNotFoundException, IOException { + SerializableMetadata met = new SerializableMetadata(ENCODING,false); + met.loadMetadataFromXmlStream(new FileInputStream(FlatDirMetadataHandler.getLocation(file,user))); + return met; + } + + /* (non-Javadoc) + * @see org.apache.oodt.cas.curation.metadata.MetadataHandler#set(java.lang.String, org.apache.oodt.cas.metadata.Metadata) + */ + @Override + public void set(String file,String user, Metadata metadata) throws FileNotFoundException, IOException { + SerializableMetadata ser = new SerializableMetadata(metadata); + ser.writeMetadataToXmlStream(new FileOutputStream(FlatDirMetadataHandler.getLocation(file,user))); + } + /* + * (non-Javadoc) + * @see org.apache.oodt.cas.curation.metadata.MetadataHandler#remove(java.lang.String) + */ + @Override + public void remove(String file,String user) throws FileNotFoundException, IOException { + File metfile = FlatDirMetadataHandler.getLocation(file,user); + if (metfile.exists()) { + if (!metfile.delete()) { + throw new IOException("Failed to delete: "+metfile.getAbsolutePath()); + } + } + } + /** + * Gets the flat-dir location for the file's metadata + * @param file - file object + * @param user - user isolation id + * @return file representing location of the flat-file + */ + private static File getLocation(String file,String user) { + return new File(Configuration.getWithReplacement(Configuration.METADATA_AREA_CONFIG),user.replace(File.separatorChar, '_')+"-"+file.replace(File.separatorChar, '_')+".met"); + } +} http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/metadata/MetadataBackend.java ---------------------------------------------------------------------- diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/metadata/MetadataBackend.java b/curator2/src/main/java/org/apache/oodt/cas/curation/metadata/MetadataBackend.java new file mode 100644 index 0000000..6bdeb59 --- /dev/null +++ b/curator2/src/main/java/org/apache/oodt/cas/curation/metadata/MetadataBackend.java @@ -0,0 +1,178 @@ +package org.apache.oodt.cas.curation.metadata; + +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.oodt.cas.curation.configuration.Configuration; +import org.apache.oodt.cas.curation.metadata.FlatDirMetadataHandler; +import org.apache.oodt.cas.curation.metadata.MetadataHandler; +import org.apache.oodt.cas.curation.structs.ExtractorConfig; +import org.apache.oodt.cas.curation.util.ExtractorConfigReader; +import org.apache.oodt.cas.metadata.MetExtractor; +import org.apache.oodt.cas.metadata.Metadata; +import org.apache.oodt.cas.metadata.exceptions.MetExtractionException; +import org.apache.oodt.cas.metadata.extractors.MetadataProvidedMetExtractor; +import org.apache.oodt.cas.metadata.util.GenericMetadataObjectFactory; + +/** + * A backend that persists the working metadata entry from the front end. + * Also handles metadata extraction, and extractor listing. + * + * @author starchmd + */ +public class MetadataBackend { + + private static final Logger LOG = Logger.getLogger(MetadataBackend.class.getName()); + private MetadataHandler handler; + private final Map<String,ExtractorConfig> extractors = new HashMap<String,ExtractorConfig>(); + /** + * Construct a directory backend with hard-coded directories + */ + public MetadataBackend() { + handler = new FlatDirMetadataHandler(); + } + /** + * Gets persisted metadata object, and runs extractor if extractor is specified + * @param file - file to get metadata from + * @param user - user used to prevent cross-talk + * @param extractor - if specified, this extractor will be run and replace existing metadata + * @return newly constructed metadat object + */ + public Metadata getMetadata(String file, String user, String extractor) throws Exception { + LOG.info("Getting metadata for: "+file+" and extractor: "+extractor); + Metadata met = null; + try { + System.out.println("Getting metadata for file: "+file +" with extractor: "+extractor); + met = handler.get(file,user); + } catch(Exception e) { + met = new Metadata(); + } + //If extractor is specified, then its metadata is considered "correct" and previous metadata is used only to fill filler + if (extractors.containsKey(extractor)) { + System.out.println("Merging"); + met = extractMergeAndPresist(file,user,extractor,met); + } + return met; + } + /** + * Persists the metadata for a given file and runs extractor, if specified + * @param file - file to specify metadata for + * @param user - user used to prevent cross-talk + * @param extractor - optional extractor to run + * @param metadata - new metadata for file + * @return metadata after optional extraction + */ + public Metadata putMetadata(String file, String user, String extractor,Metadata metadata) throws Exception { + LOG.info("Putting metadata for: "+file+" and extractor: "+extractor); + fineLogMetadata("Put metadata:",metadata); + handler.set(file, user, metadata); + if (extractors.containsKey(extractor)) { + metadata = extractMergeAndPresist(file,user,extractor,metadata); + } + return metadata; + } + /** + * Deletes the metadata for a given file + * @param file - file to specify metadata for + * @param user - user used to prevent cross-talk + */ + public void deleteMetadata(String file,String user) throws Exception { + LOG.info("Deleting metadata for: "+file); + handler.remove(file,user); + } + + /** + * Returns the list of extractors + * @return - list of extractors + */ + public Collection<String> getExtractors() throws Exception { + loadMetadataExtractors(); + return this.extractors.keySet(); + } + /** + * Loads the metadata extractors + */ + protected void loadMetadataExtractors() { + File directory = new File(Configuration.getWithReplacement(Configuration.EXTRACTOR_AREA_CONFIG)); + //Load only sub-directories of the extractor config area + FilenameFilter filter = new FilenameFilter() { + @Override + public boolean accept(File current, String name) { + return new File(current, name).isDirectory(); + } + }; + String[] subdirs = directory.list(filter); + for (String id : subdirs != null?subdirs:new String[]{}) { + try { + extractors.put(id,ExtractorConfigReader.readFromDirectory(directory, id)); + } catch (IOException e) { + LOG.log(Level.WARNING, "Failed to load extractor confugred at:"+new File(directory,id).toString()+" Exception:", e); + } + } + } + /** + * In both set and get the extractor must be run, results merged, and whole metadata persisted + * @param file - file associated with this run + * @param user - user used to prevent cross-talk + * @param extractor - extractor to run + * @param met - metadata object + * @return metadata object after merging + * @throws Exception + */ + protected Metadata extractMergeAndPresist(String file,String user,String extractor, Metadata met) throws Exception { + LOG.info("Running "+extractor+" extractoe and presisting metadat for:"+file); + System.out.println("Running Extractor"); + Metadata extracted = this.runExtractor(file, extractor,met); + System.out.println("Replace Metadata"); + met.replaceMetadata(extracted); + fineLogMetadata("Merged metadata:",met); + //Persist newly extracted metadata + System.out.println("Persisting metadata"); + handler.set(file, user, met); + return met; + } + /** + * If possible, logs metadata values finely + */ + private void fineLogMetadata(String message, Metadata metadata) { + if (LOG.isLoggable(Level.FINE)) { + for (String key : metadata.getAllKeys()) { + message += "\n\t"+key+": "+metadata.getMetadata(key); + } + LOG.fine(message); + } + } + /** + * Runs a metadata extractor on given file + * @param file - file to extract metadata from + * @param id - id of the extractor to use + * @param metadata - metadata to attach to extraction, if possible + * @return metadata extracted + * @throws MetExtractionException - exception thrown when failure to extract metadata + */ + protected Metadata runExtractor(String file, String id, Metadata metadata) throws MetExtractionException { + System.out.println("Here1"); + String parent = new File(Configuration.getWithReplacement(Configuration.STAGING_AREA_CONFIG)).getParent(); + File full = new File(parent,file); + //Get extractor + System.out.println("Here2"); + ExtractorConfig config = extractors.get(id); + System.out.println("Here3"); + MetExtractor metExtractor = GenericMetadataObjectFactory.getMetExtractorFromClassName(config.getClassName()); + System.out.println("Here4"); + metExtractor.setConfigFile(config.getConfigFiles().get(0)); + System.out.println("Here5"); + if (metExtractor instanceof MetadataProvidedMetExtractor) { + ((MetadataProvidedMetExtractor)metExtractor).attachMetadata(metadata); + } + System.out.println("Here6"); + return metExtractor.extractMetadata(full.getAbsolutePath()); + } +} http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/metadata/MetadataHandler.java ---------------------------------------------------------------------- diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/metadata/MetadataHandler.java b/curator2/src/main/java/org/apache/oodt/cas/curation/metadata/MetadataHandler.java new file mode 100644 index 0000000..05bd592 --- /dev/null +++ b/curator2/src/main/java/org/apache/oodt/cas/curation/metadata/MetadataHandler.java @@ -0,0 +1,33 @@ +package org.apache.oodt.cas.curation.metadata; + +import org.apache.oodt.cas.metadata.Metadata; + +/** + * A specific handler for metadata temporary storage. This does not handle the final cataloging, + * but acts as a temporaty storage of metadata that the user is curating. + * + * @author starchmd + */ +public interface MetadataHandler { + /** + * Gets the metadata for a file or null if none exists + * @param file - file the metadata describes + * @param user - user id to isolate work + * @return OODT metadata object + * @throws Exception + */ + public Metadata get(String file,String user) throws Exception; + /** + * Sets the metadata for a file + * @param file - file the metadata describes + * @param user - user id to isolate work + * @param metadata - metadata object to set + */ + public void set(String file,String user,Metadata metadata) throws Exception; + /** + * Remove metadata storage for file + * @param file - file the metadata describes + * @param user - user id to isolate work + */ + public void remove(String file,String user) throws Exception; +} http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/rest/DirectoryBackend.java ---------------------------------------------------------------------- diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/rest/DirectoryBackend.java b/curator2/src/main/java/org/apache/oodt/cas/curation/rest/DirectoryBackend.java new file mode 100644 index 0000000..3e0c6e5 --- /dev/null +++ b/curator2/src/main/java/org/apache/oodt/cas/curation/rest/DirectoryBackend.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oodt.cas.curation.rest; + +import org.apache.oodt.cas.curation.configuration.Configuration; +import org.apache.oodt.cas.curation.directory.Directory; +import org.apache.oodt.cas.curation.directory.FileDirectory; +import org.apache.oodt.commons.validation.DirectoryValidator; + +import com.google.gson.Gson; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.HashMap; +import java.util.Map; + +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; + +/** + * A directory backend that returns JSON object representing a directory structure. + * Will also list available directory backends. + * + * @author starchmd + */ +@Path("directory") +public class DirectoryBackend { + private DirectoryValidator validator; + Map<String,Directory> types = new HashMap<String,Directory>(); + Gson gson = new Gson(); + /** + * Construct a directory backend with hard-coded directories + */ + public DirectoryBackend() { + + } + @GET + @Produces("application/json") + /** + * Get the types of directory backends. + */ + public String getDirectoryTypes() { + //TODO: update this loading code to be user-configured and be fed off of "type" + if (types.get("files") == null) + bootstrapValidator(); + types.put("files", new FileDirectory(Configuration.getWithReplacement(Configuration.STAGING_AREA_CONFIG), + validator)); + return gson.toJson(types.keySet()); + } + + @GET + @Produces("application/json") + @Path("{type}") + /** + * Returns the listing of the given directory type + * @param type - type of directory to list + */ + public String list(@PathParam("type") String type) throws Exception { + //TODO: update this loading code to be user-configured and be fed off of "type" + if (types.get("files") == null) + bootstrapValidator(); + types.put("files", new FileDirectory(Configuration.getWithReplacement(Configuration.STAGING_AREA_CONFIG), + validator)); + return gson.toJson(types.get(type).list()); + } + + /** + * Initialise the validator engine as defined in web.xml. + */ + private void bootstrapValidator(){ + if(validator==null) { + String vclass = Configuration.getWithReplacement(Configuration.DIRECTORYBACKEND_VALIDATOR); + if (vclass != null && !vclass.equals("")) { + Class<?> clazz = null; + try { + clazz = Class.forName(vclass); + + Constructor<?> constructor = clazz.getConstructor(); + this.validator = (DirectoryValidator) constructor.newInstance(); + + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } catch (NoSuchMethodException e) { + e.printStackTrace(); + } catch (InstantiationException e) { + e.printStackTrace(); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } catch (InvocationTargetException e) { + e.printStackTrace(); + } + } + } + } +} http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/rest/ExceptionResponseHandler.java ---------------------------------------------------------------------- diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/rest/ExceptionResponseHandler.java b/curator2/src/main/java/org/apache/oodt/cas/curation/rest/ExceptionResponseHandler.java new file mode 100644 index 0000000..edfd898 --- /dev/null +++ b/curator2/src/main/java/org/apache/oodt/cas/curation/rest/ExceptionResponseHandler.java @@ -0,0 +1,52 @@ +package org.apache.oodt.cas.curation.rest; + +import javax.ws.rs.core.Response; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +/** + * Helper class to build response for thrown exception + * + * @author starchmd + */ +public class ExceptionResponseHandler { + /* + * Static initializer to get GSON converter + */ + private static Gson gson; + { + gson = new GsonBuilder().create(); + } + /** + * Recurse down an exception causality tree + * @param e -exception + * @return exception info regarding this exception + */ + public static ExceptionInfo recurseExceptionTree(Throwable e) { + ExceptionInfo info = new ExceptionInfo(); + info.name = e.getClass().getName(); + info.message = e.getLocalizedMessage(); + info.stack = e.getStackTrace(); + info.cause = recurseExceptionTree(e.getCause()); + return info; + } + /** + * Return a response that wraps an exception + * @return + */ + public static Response BuildExceptionResponse(Exception e) { + ExceptionInfo info = recurseExceptionTree(e); + return Response.serverError().entity(gson.toJson(info)).build(); + } + /** + * Information regarding an exception + * @author starchmd + */ + static class ExceptionInfo { + String name; + String message; + ExceptionInfo cause; + StackTraceElement[] stack; + } +} http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/rest/IngestRest.java ---------------------------------------------------------------------- diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/rest/IngestRest.java b/curator2/src/main/java/org/apache/oodt/cas/curation/rest/IngestRest.java new file mode 100644 index 0000000..6efb563 --- /dev/null +++ b/curator2/src/main/java/org/apache/oodt/cas/curation/rest/IngestRest.java @@ -0,0 +1,111 @@ +package org.apache.oodt.cas.curation.rest; + + +import java.util.LinkedList; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Response; +import javax.ws.rs.Produces; +import javax.ws.rs.GET; + +import org.apache.commons.lang.StringUtils; +import org.apache.oodt.cas.curation.ingest.IngestBackend; +import org.apache.oodt.cas.curation.ingest.IngestException; +import org.apache.oodt.cas.curation.ingest.InputStruct; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +/** + * A backend that uses put to initiate ingest + * + * @author starchmd + */ +@Path("ingest") +public class IngestRest { + private static final Logger LOG = Logger.getLogger(IngestRest.class.getName()); + //GSON serialization object using Google GSON + private Gson gson = new GsonBuilder().create(); + private IngestBackend backend = null; + /** + * Construct a directory backend with hard-coded directories + */ + public IngestRest() {} + @PUT + @Consumes("application/json") + /** + * Runs the ingest + * @param user - user to isolate requests + * @param extractor - optional extractor to run + * @param json - new json for file + */ + public Response ingest(@QueryParam("user") String user,@QueryParam("extractor") String extractor,String json) { + try { + setup(); + InputStruct input = gson.fromJson(json, InputStruct.class); + List<String> files = new LinkedList<String>(); + for (InputStruct.InputEntry entry : input.entries) { + files.add(entry.file); + } + LOG.log(Level.INFO, "Ingesting files: "+StringUtils.join(files,",")); + this.backend.ingest(input,user); + return Response.ok().build(); + } catch (Exception e) { + LOG.log(Level.SEVERE,"Exception occured calling ingest REST endpoint", e); + return ExceptionResponseHandler.BuildExceptionResponse(e); + } + } + @GET + @Produces("application/json") + /** + * Get ingest status + * @return list of statuses + */ + public Response status(@QueryParam("user") String user) { + try { + setup(); + LOG.log(Level.INFO, "Reading current status"); + return Response.ok().entity(gson.toJson(this.backend.status(user))).build(); + } catch(Exception e) { + LOG.log(Level.SEVERE,"Exception occured calling ingest REST endpoint", e); + return ExceptionResponseHandler.BuildExceptionResponse(e); + } + } + @DELETE + /** + * Delete errors + * @return response object + */ + public Response remove(@QueryParam("user") String user) { + try { + setup(); + LOG.log(Level.INFO, "Deleteing current errors"); + this.backend.clearErrors(user); + return Response.ok().build(); + } catch(Exception e) { + LOG.log(Level.SEVERE,"Exception occured calling ingest REST endpoint", e); + return ExceptionResponseHandler.BuildExceptionResponse(e); + } + } + /** + * Setup the backend + * @throws IngestException + */ + public synchronized void setup() throws IngestException { + if (this.backend != null) + return; + LOG.log(Level.INFO, "Setting up ingest backend"); + this.backend = new IngestBackend(); + } + + public IngestBackend getBackend() { + return backend; + } +} http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/rest/MetadataRest.java ---------------------------------------------------------------------- diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/rest/MetadataRest.java b/curator2/src/main/java/org/apache/oodt/cas/curation/rest/MetadataRest.java new file mode 100644 index 0000000..8c65a42 --- /dev/null +++ b/curator2/src/main/java/org/apache/oodt/cas/curation/rest/MetadataRest.java @@ -0,0 +1,139 @@ +package org.apache.oodt.cas.curation.rest; + +import java.util.logging.Logger; + +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.QueryParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Response; + +import org.apache.oodt.cas.curation.metadata.MetadataBackend; +import org.apache.oodt.cas.metadata.Metadata; + +import com.google.gson.ExclusionStrategy; +import com.google.gson.FieldAttributes; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +/** + * A backend that puts/gets the metadata of a given file. + * + * @author starchmd + */ +@Path("metadata") +public class MetadataRest { + + private static final Logger LOG = Logger.getLogger(MetadataRest.class.getName()); + //GSON serialization object using Google GSON + private Gson gson = new GsonBuilder().setExclusionStrategies(new ExclusionStrategy(){ + @Override + public boolean shouldSkipClass(Class<?> clazz) { + return false; + } + @Override + public boolean shouldSkipField(FieldAttributes field) { + return field.getName().equals("parent"); + }}).create(); + private MetadataBackend backend; + + /** + * Construct a directory backend with hard-coded directories + */ + public MetadataRest() {} + @GET + @Produces("application/json") + @Path("{file:.+}") + /** + * Gets the metadata as JSON, refreshes using an extractor + * @param file - file to get metadata from + * @param extractor - if specified, this extractor will be run and replace existing metadata + */ + public Response getMetadata(@PathParam("file") String file,@QueryParam("user") String user,@QueryParam("extractor") String extractor) { + try { + if (file.equals("dev/null")) + { + file = "/dev/null"; + } + this.setup(); + System.out.println("Getting metadata for file: "+file); + Metadata met = this.backend.getMetadata(file,(user==null)?"":user, extractor); + return Response.ok().entity(gson.toJson(met)).build(); + } catch (Exception e) { + return ExceptionResponseHandler.BuildExceptionResponse(e); + } + } + + @PUT + @Consumes("application/json") + @Produces("application/json") + @Path("{file:.+}") + /** + * Sets the metadata for a given file + * @param file - file to specify metadata for + * @param extractor - optional extractor to run + * @param json - new json for file + */ + public Response putMetadata(@PathParam("file") String file,@QueryParam("user") String user,@QueryParam("extractor") String extractor,String json) { + try { + if (file.equals("dev/null")) + { + file = "/dev/null"; + } + Metadata input = gson.fromJson(json, Metadata.class); + this.setup(); + Metadata met = this.backend.putMetadata(file,(user==null)?"":user, extractor, input); + return Response.ok().entity(gson.toJson(met)).build(); + } catch(Exception e) { + return ExceptionResponseHandler.BuildExceptionResponse(e); + + } + } + @DELETE + @Path("{file:.+}") + /** + * Deletes the metadata for a given file + * @param file - file to specify metadata for + */ + public Response deleteMetadata(@PathParam("file") String file,@QueryParam("user") String user) { + try { + if (file.equals("dev/null")) + { + file = "/dev/null"; + } + this.setup(); + this.backend.deleteMetadata(file,(user==null)?"":user); + return Response.ok().build(); + } catch(Exception e) { + return ExceptionResponseHandler.BuildExceptionResponse(e); + } + } + @GET + @Produces("application/json") + @Path("extractors") + /** + * Returns the list of extractors + * @return - list of extractors + */ + public Response getExtractors() { + try { + this.setup(); + return Response.ok().entity(gson.toJson(this.backend.getExtractors())).build(); + } catch (Exception e) { + return ExceptionResponseHandler.BuildExceptionResponse(e); + } + } + /** + * Setup the backend to call + */ + private synchronized void setup() { + if (this.backend != null) + return; + LOG.info("Setting up metadata backend for metadata rest service"); + this.backend = new MetadataBackend(); + } +} http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/rest/UploadBackend.java ---------------------------------------------------------------------- diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/rest/UploadBackend.java b/curator2/src/main/java/org/apache/oodt/cas/curation/rest/UploadBackend.java new file mode 100644 index 0000000..07f3633 --- /dev/null +++ b/curator2/src/main/java/org/apache/oodt/cas/curation/rest/UploadBackend.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oodt.cas.curation.rest; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.List; +import java.util.logging.Level; + +import javax.ws.rs.Consumes; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import org.apache.commons.io.IOUtils; +import org.apache.cxf.jaxrs.ext.multipart.Attachment; +import org.apache.cxf.jaxrs.ext.multipart.MultipartBody; +import org.apache.oodt.cas.curation.configuration.Configuration; + +/** + * A RESTful backend allowing for the user to upload files + * to the CAS-Curator. + * + * @author starchmd + */ +@Path("upload") +public class UploadBackend { + /** + * Construct using configuration to determine upload area + */ + public UploadBackend() {} + + @POST + @Path("file") + @Consumes(MediaType.MULTIPART_FORM_DATA) + /** + * Handles the uploading of the data, by copying all data out of the input stream. + * @param mbody - body of the multipart post + * @return - OK response on success + */ + public Response upload(MultipartBody mbody) { + try { + List<Attachment> attachments = mbody.getAllAttachments(); + for (Attachment attachment : attachments) { + String filename = attachment.getContentDisposition().getParameter("filename"); + try { + InputStream in = attachment.getDataHandler().getInputStream(); + OutputStream os = new FileOutputStream(new File(Configuration.getWithReplacement(Configuration.UPLOAD_AREA_CONFIG),filename)); + IOUtils.copy(in,os); + in.close(); + os.close(); + } catch(IOException e) { + throw new IOException("Failed uploading file:",e); + } + } + return Response.ok().build(); + } catch(Exception e) { + return ExceptionResponseHandler.BuildExceptionResponse(e); + } + } +} http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/rest/ValidationRest.java ---------------------------------------------------------------------- diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/rest/ValidationRest.java b/curator2/src/main/java/org/apache/oodt/cas/curation/rest/ValidationRest.java new file mode 100644 index 0000000..0f83d2d --- /dev/null +++ b/curator2/src/main/java/org/apache/oodt/cas/curation/rest/ValidationRest.java @@ -0,0 +1,65 @@ +package org.apache.oodt.cas.curation.rest; + +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Response; + +import org.apache.oodt.cas.curation.configuration.Configuration; +import org.apache.oodt.cas.curation.validation.ValidationBackend; +import org.apache.oodt.cas.curation.validation.ValidationException; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + + +/** + * A backend that loads and prepares a validation layer from the + * filemanager. + * + * @author starchmd + */ +@Path("validation") +public class ValidationRest { + private static final Logger LOG = Logger.getLogger(ValidationBackend.class.getName()); + private Gson gson = new GsonBuilder().create(); + ValidationBackend backend = null; + + /** + * Empty REST service constructor. (Configuration not available yet. + */ + public ValidationRest(){} + + @GET + @Produces("application/json") + /** + * Return information pertaining to validation + * @return - OK response on success + */ + public Response information() { + try { + setup(); + return Response.ok().entity(gson.toJson(this.backend.getValidation())).build(); + } catch(Exception e) { + LOG.log(Level.SEVERE,"Validation exception occured calling validation/info REST endpoint", e); + return ExceptionResponseHandler.BuildExceptionResponse(e); + } + } + /** + * Setup this service + */ + public synchronized void setup() throws ValidationException { + if (this.backend != null) + return; + try { + LOG.log(Level.INFO, "Setting up validation backend"); + this.backend = new ValidationBackend(Configuration.getWithReplacement(Configuration.FILEMANAGER_PROP_CONFIG)); + } catch(ValidationException ve) { + LOG.log(Level.SEVERE,"Validation exception occured setting up validation REST service", ve); + throw ve; + } + } +} http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/service/CurationService.java ---------------------------------------------------------------------- diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/service/CurationService.java b/curator2/src/main/java/org/apache/oodt/cas/curation/service/CurationService.java new file mode 100644 index 0000000..903d65d --- /dev/null +++ b/curator2/src/main/java/org/apache/oodt/cas/curation/service/CurationService.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.oodt.cas.curation.service; + +//OODT imports +import org.apache.oodt.cas.curation.servlet.CuratorConfMetKeys; +import org.apache.oodt.cas.curation.util.SSOUtils; +import org.apache.oodt.security.sso.SingleSignOn; + +//JDK imports +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; +import javax.servlet.ServletConfig; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.core.UriInfo; + +//JAX-RS imports +import net.sf.json.JSONArray; +import net.sf.json.JSONObject; + +//APACHE imports +import org.apache.commons.lang.StringUtils; + +/** + * + * + * The Main Curation Web Service responsible for providing basic tools and + * functionality for performing ingestion into the CAS filemgr, and management + * of the staging area for ingestion. + * + * @author pramirez + * @author mattmann + * @version $Revision$ + * + */ +public class CurationService extends HttpServlet implements CuratorConfMetKeys { + + private static final long serialVersionUID = -5370697580594691669L; + + private static final Logger LOG = Logger.getLogger(CurationService.class + .getName()); + + protected static CurationServiceConfig config = null; + + protected final static String FORMAT_JSON = "json"; + + protected final static String FORMAT_HTML = "html"; + + protected final static String UNKNOWN_OUT_FORMAT = "Unsupported Output Format!"; + + protected SingleSignOn sso; + + /** + * Default Constructor. + */ + public CurationService() { + } + + @Override + public void init(ServletConfig conf) throws ServletException { + super.init(conf); + this.config = CurationServiceConfig.getInstance(conf); + } + + /** + * Gets the staging area as a JSON formatted response object. + * + * @param base + * The base directory to read files from, should be the staging area + * root. + * @param path + * The particular child path within the staging area. + * @param showFiles + * Whether or not to show {@link File#isFile()} files or not. + * @return A String representation formatting using + * {@link JSONObject#toString()}. + */ + public String getDirectoryAreaAsJSON(String base, String path, + boolean showFiles) { + String startingPath = (base + "/" + path); + startingPath = StringUtils.replace(startingPath, "source", "/"); + String f[] = getFilesInDirectory(startingPath, showFiles); + + List<Map<String, Object>> items = new ArrayList<Map<String, Object>>(); + for (int i = 0; i < f.length; i++) { + Map<String, Object> entry = new HashMap<String, Object>(); + String children[] = getFilesInDirectory(startingPath + "/" + f[i], + showFiles); + entry.put("text", f[i]); + entry.put("id", path + "/" + f[i]); + entry.put("expanded", false); + entry.put("hasChildren", children != null && (children.length > 0)); + entry.put("isFile", new File(startingPath + "/" + f[i]).isFile()); + items.add(entry); + } + + return JSONArray.fromObject(items).toString(); + } + + protected String[] getFilesInDirectory(String directory, + final boolean showFiles) { + File dir = new File(directory); + FilenameFilter filter = new FilenameFilter() { + + public boolean accept(File dir, String name) { + + return !name.startsWith(".") + && (showFiles || !new File(dir, name).isFile()); + } + }; + return dir.list(filter); + } + + protected void sendRedirect(String page, UriInfo uriInfo, + HttpServletResponse response) throws IOException { + response.sendRedirect(uriInfo.getBaseUriBuilder().path("../" + page) + .build().toASCIIString()); + } + + protected String cleansePath(String path) throws UnsupportedEncodingException { + String newPath = path; + if (newPath.startsWith("/")) { + newPath = path.substring(1); + } + newPath = URLDecoder.decode(newPath, "UTF-8"); + return newPath; + + } + + /** + * Configures the web context persistence layer for the CAS SSO so that all + * services ({@link HttpServlet}s) that extend this implementation get the + * ability to configure an SSO object for free, essentially. + * + * @param req + * The HTTP request object. + * @param res + * The HTTP response object. + */ + protected void configureSingleSignOn(HttpServletRequest req, + HttpServletResponse res) { + this.sso = SSOUtils.getWebSingleSignOn(CurationService.config, req, res); + } +}
