Repository: oodt Updated Branches: refs/heads/feature/zookeeper-config ec5b20133 -> 3fb76606a
Fix for OODT-847 contributed by adhulipa & rverma Implemented FileAttributesExtractor which leverages the Java 7 API to get file system attributes. Project: http://git-wip-us.apache.org/repos/asf/oodt/repo Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/ce690461 Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/ce690461 Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/ce690461 Branch: refs/heads/feature/zookeeper-config Commit: ce690461b05da324ce1cb5d480409adec6737737 Parents: 0eacdfe Author: Aditya Dhulipala <[email protected]> Authored: Fri Oct 28 05:36:13 2016 -0400 Committer: Aditya Dhulipala <[email protected]> Committed: Fri Oct 28 05:36:13 2016 -0400 ---------------------------------------------------------------------- .../filemgr/metadata/FileAttributesMetKeys.java | 53 ++++++ .../examples/FileAttributesExtractor.java | 82 +++++++++ .../examples/TestFileAttributesExtractor.java | 167 +++++++++++++++++++ 3 files changed, 302 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oodt/blob/ce690461/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/FileAttributesMetKeys.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/FileAttributesMetKeys.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/FileAttributesMetKeys.java new file mode 100644 index 0000000..30ca080 --- /dev/null +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/FileAttributesMetKeys.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oodt.cas.filemgr.metadata; + +/** + * Met key field names used to augment {@link Product} {@link Metadata}. + * @author adhulipala + * @version $Revision$ + * @since OODT-847 + */ +public interface FileAttributesMetKeys { + + // Basic file attributes + String IS_SYMBOLIC_LINK = "isSymbolicLink"; + + String CREATION_TIME = "creationTime"; + + String LAST_MODIFIED_TIME = "lastModifiedTime"; + + String IS_OTHER = "isOther"; + + String IS_DIRECTORY = "isDirectory"; + + String FILE_KEY = "fileKey"; + + String LAST_ACCESS_TIME = "lastAccessTime"; + + String IS_REGULAR_FILE = "isRegularFile"; + + String SIZE = "size"; + + // POSIX File attributes + String OWNER = "owner"; + + String PERMISSIONS = "permissions"; + + String GROUP = "group"; +} http://git-wip-us.apache.org/repos/asf/oodt/blob/ce690461/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/extractors/examples/FileAttributesExtractor.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/extractors/examples/FileAttributesExtractor.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/extractors/examples/FileAttributesExtractor.java new file mode 100644 index 0000000..f6a0490 --- /dev/null +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/extractors/examples/FileAttributesExtractor.java @@ -0,0 +1,82 @@ +package org.apache.oodt.cas.filemgr.metadata.extractors.examples; + +import org.apache.oodt.cas.filemgr.metadata.extractors.AbstractFilemgrMetExtractor; +import org.apache.oodt.cas.filemgr.structs.Product; +import org.apache.oodt.cas.metadata.Metadata; +import org.apache.oodt.cas.metadata.exceptions.MetExtractionException; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Logger; + +/** + * @author rverma + * @author adhulipala + */ +public class FileAttributesExtractor extends AbstractFilemgrMetExtractor { + + public static final String BASIC_FILE_ATTRIBUTES = "*"; + public static final String POSIX_FILE_ATTRIBUTES = "posix:*"; + + private String attributes; + + Logger LOG = Logger.getLogger(FileAttributesExtractor.class.getName()); + + public FileAttributesExtractor() { + attributes = BASIC_FILE_ATTRIBUTES; + } + + public FileAttributesExtractor(String attributes) { + this.attributes = attributes; + } + + @Override + public Metadata doExtract(Product product, Metadata metadata) throws MetExtractionException { + Metadata outMetadata = new Metadata(); + + merge(metadata, outMetadata); + Metadata fileAttributesMetadata = getMetadataFromFileAttributes(product); + + LOG.fine(fileAttributesMetadata.toString()); + + merge(fileAttributesMetadata, outMetadata); + + return outMetadata; + } + + @Override + public void doConfigure() { + if (this.configuration != null) { + this.attributes = this.configuration.getProperty("attributes"); + } + } + + + private Metadata getMetadataFromFileAttributes(Product product) { + Metadata met = new Metadata(); + + File file = null; + try { + file = getProductFile(product); + } catch (MetExtractionException e) { + LOG.severe(e.getMessage()); + } + + if (file != null) { + Map<String, Object> attrMap = new HashMap<String, Object>(); + try { + attrMap = Files.readAttributes(file.toPath(), attributes); + } catch (IOException e) { + LOG.severe(e.getMessage()); + } + + for (Map.Entry<String, Object> entry : attrMap.entrySet()) { + met.addMetadata(entry.getKey(), String.valueOf(entry.getValue())); + } + } + return met; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oodt/blob/ce690461/filemgr/src/test/java/org/apache/oodt/cas/filemgr/metadata/extractors/examples/TestFileAttributesExtractor.java ---------------------------------------------------------------------- diff --git a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/metadata/extractors/examples/TestFileAttributesExtractor.java b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/metadata/extractors/examples/TestFileAttributesExtractor.java new file mode 100644 index 0000000..e425c23 --- /dev/null +++ b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/metadata/extractors/examples/TestFileAttributesExtractor.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oodt.cas.filemgr.metadata.extractors.examples; + +//JDK imports +import java.net.URL; +import java.util.Properties; + +//OODT imports +import org.apache.oodt.cas.filemgr.metadata.FileAttributesMetKeys; +import org.apache.oodt.cas.filemgr.structs.Product; +import org.apache.oodt.cas.filemgr.structs.Reference; +import org.apache.oodt.cas.metadata.Metadata; +import org.apache.oodt.cas.metadata.exceptions.MetExtractionException; + +//Junit imports +import junit.framework.TestCase; + +/** + * @author adhulipala + * @version $Revision$ + * @since OODT-847 + * <p> + * Test suite for the {@link FileAttributesExtractor}. + * </p>. + */ +public class TestFileAttributesExtractor extends TestCase implements FileAttributesMetKeys { + + private final FileAttributesExtractor fileAttributesExtractor = new FileAttributesExtractor(); + + public void testExtractBasicFileAttributes() { + // Define Reference & Product + URL refUrl = this.getClass().getResource("/ingest/test.txt"); + Reference ref = new Reference(); + ref.setOrigReference(refUrl.toString()); + ref.setDataStoreReference(refUrl.toString()); + + Product product = new Product(); + product.getProductReferences().add(ref); + product.setProductStructure(Product.STRUCTURE_FLAT); + + // Define configuration to extract basic file attributes + Properties config = new Properties(); + config.setProperty("attributes", "*"); + + // Configure Extractor + fileAttributesExtractor.configure(config); + + // Extract file attributes metadata + Metadata metadata = new Metadata(); + + try { + metadata = fileAttributesExtractor.doExtract(product, metadata); + + } catch (MetExtractionException e) { + fail(e.getMessage()); + } + + // Test + assertNotNull(metadata); + assertTrue(metadata.containsKey(IS_SYMBOLIC_LINK)); + assertTrue(metadata.containsKey(CREATION_TIME)); + assertTrue(metadata.containsKey(LAST_MODIFIED_TIME)); + assertTrue(metadata.containsKey(IS_OTHER)); + assertTrue(metadata.containsKey(IS_DIRECTORY)); + assertTrue(metadata.containsKey(FILE_KEY)); + assertTrue(metadata.containsKey(LAST_ACCESS_TIME)); + assertTrue(metadata.containsKey(IS_REGULAR_FILE)); + assertTrue(metadata.containsKey(SIZE)); + } + + public void testExtractPosixFileAttributes() { + // Define Reference & Product + URL refUrl = this.getClass().getResource("/ingest/test.txt"); + Reference ref = new Reference(); + ref.setOrigReference(refUrl.toString()); + ref.setDataStoreReference(refUrl.toString()); + + Product product = new Product(); + product.getProductReferences().add(ref); + product.setProductStructure(Product.STRUCTURE_FLAT); + + // Define configuration to extract basic file attributes + Properties config = new Properties(); + config.setProperty("attributes", "posix:*"); + + // Configure Extractor + fileAttributesExtractor.configure(config); + + // Extract file attributes metadata + Metadata metadata = new Metadata(); + + try { + metadata = fileAttributesExtractor.doExtract(product, metadata); + } catch (MetExtractionException e) { + fail(e.getMessage()); + } + + // Test + assertNotNull(metadata); + assertTrue(metadata.containsKey(OWNER)); + assertTrue(metadata.containsKey(PERMISSIONS)); + assertTrue(metadata.containsKey(GROUP)); + } + + public void testExtractSpecificFileAttributes() { + // Define Reference & Product + URL refUrl = this.getClass().getResource("/ingest/test.txt"); + Reference ref = new Reference(); + ref.setOrigReference(refUrl.toString()); + ref.setDataStoreReference(refUrl.toString()); + + Product product = new Product(); + product.getProductReferences().add(ref); + product.setProductStructure(Product.STRUCTURE_FLAT); + + // Define configuration to extract basic file attributes + Properties config = new Properties(); + config.setProperty("attributes", "posix:size,owner,creationTime"); + + // Configure Extractor + fileAttributesExtractor.configure(config); + + // Extract file attributes metadata + Metadata metadata = new Metadata(); + + try { + metadata = fileAttributesExtractor.doExtract(product, metadata); + } catch (MetExtractionException e) { + fail(e.getMessage()); + } + + System.out.println(metadata.getAllKeys()); + System.out.println(metadata.containsKey(OWNER)); + System.out.println(metadata.containsKey(IS_SYMBOLIC_LINK)); + + // Test presence of expected attributes + assertNotNull(metadata); + assertTrue(metadata.containsKey(OWNER)); + assertTrue(metadata.containsKey(SIZE)); + assertTrue(metadata.containsKey(CREATION_TIME)); + + // Test absence of other attributes + assertFalse(metadata.containsKey(IS_SYMBOLIC_LINK)); + assertFalse(metadata.containsKey(LAST_MODIFIED_TIME)); + assertFalse(metadata.containsKey(IS_OTHER)); + assertFalse(metadata.containsKey(IS_DIRECTORY)); + assertFalse(metadata.containsKey(FILE_KEY)); + assertFalse(metadata.containsKey(LAST_ACCESS_TIME)); + assertFalse(metadata.containsKey(IS_REGULAR_FILE)); + + } +}
