Author: bfoster
Date: Tue Apr 29 08:46:30 2014
New Revision: 1590919
URL: http://svn.apache.org/r1590919
Log:
- Add Metadata Extractor which uses filename to lookup metadata from an sql
based database to generate metadata.
Added:
oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/extractors/DataSourceMetExtractor.java
(with props)
oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/extractors/DataSourceMetExtractorConfig.java
(with props)
oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/extractors/DataSourceMetExtractorConfigReader.java
(with props)
oodt/trunk/metadata/src/main/resources/examples/datasource_metextractor_example.properties
(with props)
oodt/trunk/metadata/src/test/org/apache/oodt/cas/metadata/extractors/TestDataSourceMetExtractor.java
(with props)
Modified:
oodt/trunk/metadata/pom.xml
oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/exceptions/MetExtractorConfigReaderException.java
Modified: oodt/trunk/metadata/pom.xml
URL:
http://svn.apache.org/viewvc/oodt/trunk/metadata/pom.xml?rev=1590919&r1=1590918&r2=1590919&view=diff
==============================================================================
--- oodt/trunk/metadata/pom.xml (original)
+++ oodt/trunk/metadata/pom.xml Tue Apr 29 08:46:30 2014
@@ -178,9 +178,26 @@ the License.
</exclusions>
</dependency>
<dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>10.0.1</version>
+ </dependency>
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- <version>3.8.2</version>
+ <version>4.8.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>1.9.5</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ <version>1.3</version>
<scope>test</scope>
</dependency>
</dependencies>
Modified:
oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/exceptions/MetExtractorConfigReaderException.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/exceptions/MetExtractorConfigReaderException.java?rev=1590919&r1=1590918&r2=1590919&view=diff
==============================================================================
---
oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/exceptions/MetExtractorConfigReaderException.java
(original)
+++
oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/exceptions/MetExtractorConfigReaderException.java
Tue Apr 29 08:46:30 2014
@@ -14,17 +14,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-
package org.apache.oodt.cas.metadata.exceptions;
/**
- *
- * @author bfoster
- * @version $Revision$
+ * Exception throw when error occurs while parsing a config {@link File} into a
+ * {@link MetExtractorConfig}
*
- * <p>Exception throw when error occurs while parsing a config
- * {@link File} into a {@link MetExtractorConfig}</p>.
+ * @author [email protected] (Brian Foster)
*/
public class MetExtractorConfigReaderException extends Exception {
@@ -37,4 +33,8 @@ public class MetExtractorConfigReaderExc
public MetExtractorConfigReaderException(String msg) {
super(msg);
}
+
+ public MetExtractorConfigReaderException(String msg, Throwable t) {
+ super(msg, t);
+ }
}
Added:
oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/extractors/DataSourceMetExtractor.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/extractors/DataSourceMetExtractor.java?rev=1590919&view=auto
==============================================================================
---
oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/extractors/DataSourceMetExtractor.java
(added)
+++
oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/extractors/DataSourceMetExtractor.java
Tue Apr 29 08:46:30 2014
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.metadata.extractors;
+
+// JDK imports
+import java.io.File;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+// JAVAX imports
+import javax.sql.DataSource;
+
+// OODT imports
+import org.apache.oodt.cas.metadata.AbstractMetExtractor;
+import org.apache.oodt.cas.metadata.Metadata;
+import org.apache.oodt.cas.metadata.exceptions.MetExtractionException;
+import org.apache.oodt.commons.database.DatabaseConnectionBuilder;
+
+// Google imports
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Splitter;
+
+/**
+ * MetExtractor which uses input file's name as key for lookup into a sql
database to get metadata.
+ *
+ * @author [email protected] (Brian Foster)
+ */
+public class DataSourceMetExtractor extends AbstractMetExtractor {
+
+ public DataSourceMetExtractor() {
+ super(new DataSourceMetExtractorConfigReader());
+ }
+
+ @Override
+ protected Metadata extrMetadata(File file) throws MetExtractionException {
+ String key = getKey(file);
+ DataSourceMetExtractorConfig dsConfig = (DataSourceMetExtractorConfig)
config;
+ DataSource dataSource =
DatabaseConnectionBuilder.buildDataSource(dsConfig.getUserName(),
+ dsConfig.getPassword(), dsConfig.getDriver(),
dsConfig.getDatabaseUrl());
+
+ return getMetadata(dataSource, dsConfig.getQuery(), key);
+ }
+
+ @VisibleForTesting
+ protected String getKey(File file) {
+ return Splitter.on(".").split(file.getName()).iterator().next();
+ }
+
+ @VisibleForTesting
+ protected Metadata getMetadata(DataSource dataSource, String query, String
key)
+ throws MetExtractionException {
+ String sqlQuery = String.format(query, key);
+
+ Connection conn = null;
+ Statement statement = null;
+ ResultSet rs = null;
+
+ try {
+ conn = dataSource.getConnection();
+ statement = conn.createStatement();
+ rs = statement.executeQuery(sqlQuery);
+
+ return getMetadata(rs);
+ } catch (SQLException e) {
+ throw new MetExtractionException(
+ String.format("Failed to get metadaata for key '%s'", key), e);
+ } finally {
+ try { conn.close(); } catch (Exception e) { /* ignore */ }
+ try { statement.close(); } catch (Exception e) { /* ignore */ }
+ try { rs.close(); } catch (Exception e) { /* ignore */ }
+ }
+ }
+
+ private Metadata getMetadata(ResultSet rs) throws SQLException {
+ Metadata metadata = new Metadata();
+ for (int i = 0; i < rs.getMetaData().getColumnCount(); i++) {
+ String metKey = rs.getMetaData().getColumnName(i);
+ String metVal = rs.getString(i);
+ metadata.addMetadata(metKey, metVal);
+ }
+ return metadata;
+ }
+}
Propchange:
oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/extractors/DataSourceMetExtractor.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added:
oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/extractors/DataSourceMetExtractorConfig.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/extractors/DataSourceMetExtractorConfig.java?rev=1590919&view=auto
==============================================================================
---
oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/extractors/DataSourceMetExtractorConfig.java
(added)
+++
oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/extractors/DataSourceMetExtractorConfig.java
Tue Apr 29 08:46:30 2014
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.metadata.extractors;
+
+// JDK imports
+import java.util.Properties;
+
+// OODT imports
+import org.apache.oodt.cas.metadata.MetExtractorConfig;
+
+/**
+ * MetExtractorConfig which loads configuration from a Java Property file.
Expects given keys
+ * to be set and provides easy read methods for those keys.
+ *
+ * @author [email protected] (Brian Foster)
+ */
+public class DataSourceMetExtractorConfig extends Properties implements
MetExtractorConfig {
+
+ private static final long serialVersionUID = -437948882353764454L;
+
+ private static final String QUERY_KEY =
+ "org.apache.oodt.cas.metadata.extractors.datasource.query";
+ private static final String DATABASE_URL_KEY =
+ "org.apache.oodt.cas.metadata.extractors.datasource.db.url";
+ private static final String DRIVER_KEY =
+ "org.apache.oodt.cas.metadata.extractors.datasource.driver";
+ private static final String USER_NAME_KEY =
+ "org.apache.oodt.cas.metadata.extractors.datasource.username";
+ private static final String PASSWORD_KEY =
+ "org.apache.oodt.cas.metadata.extractors.datasource.password";
+
+ public String getQuery() {
+ return getProperty(QUERY_KEY);
+ }
+
+ public String getDatabaseUrl() {
+ return getProperty(DATABASE_URL_KEY);
+ }
+
+ public String getDriver() {
+ return getProperty(DRIVER_KEY);
+ }
+
+ public String getUserName() {
+ return getProperty(USER_NAME_KEY);
+ }
+
+ public String getPassword() {
+ return getProperty(PASSWORD_KEY);
+ }
+}
Propchange:
oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/extractors/DataSourceMetExtractorConfig.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added:
oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/extractors/DataSourceMetExtractorConfigReader.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/extractors/DataSourceMetExtractorConfigReader.java?rev=1590919&view=auto
==============================================================================
---
oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/extractors/DataSourceMetExtractorConfigReader.java
(added)
+++
oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/extractors/DataSourceMetExtractorConfigReader.java
Tue Apr 29 08:46:30 2014
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.metadata.extractors;
+
+// JDK imports
+import java.io.File;
+
+// OODT imports
+import org.apache.oodt.cas.metadata.MetExtractorConfigReader;
+import
org.apache.oodt.cas.metadata.exceptions.MetExtractorConfigReaderException;
+
+/**
+ * {@link MetExtractorConfigReader} which reads configuration from a Java
Properties file.
+ *
+ * @author [email protected] (Brian Foster)
+ */
+public class DataSourceMetExtractorConfigReader implements
MetExtractorConfigReader {
+
+ @Override
+ public DataSourceMetExtractorConfig parseConfigFile(File configFile)
+ throws MetExtractorConfigReaderException {
+ try {
+ DataSourceMetExtractorConfig config = new DataSourceMetExtractorConfig();
+ config.load(configFile.toURI().toURL().openStream());
+ return config;
+ } catch (Exception e) {
+ throw new MetExtractorConfigReaderException(
+ String.format("Failed to parse '%s'", configFile), e);
+ }
+ }
+}
Propchange:
oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/extractors/DataSourceMetExtractorConfigReader.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added:
oodt/trunk/metadata/src/main/resources/examples/datasource_metextractor_example.properties
URL:
http://svn.apache.org/viewvc/oodt/trunk/metadata/src/main/resources/examples/datasource_metextractor_example.properties?rev=1590919&view=auto
==============================================================================
---
oodt/trunk/metadata/src/main/resources/examples/datasource_metextractor_example.properties
(added)
+++
oodt/trunk/metadata/src/main/resources/examples/datasource_metextractor_example.properties
Tue Apr 29 08:46:30 2014
@@ -0,0 +1,5 @@
+org.apache.oodt.cas.metadata.extractors.datasource.query=select
ProductName,FileSize from Products where ProductId = '%s'
+org.apache.oodt.cas.metadata.extractors.datasource.db.url=http://db.url
+org.apache.oodt.cas.metadata.extractors.datasource.driver=some.driver.class
+org.apache.oodt.cas.metadata.extractors.datasource.username=user
+org.apache.oodt.cas.metadata.extractors.datasource.password=pass
\ No newline at end of file
Propchange:
oodt/trunk/metadata/src/main/resources/examples/datasource_metextractor_example.properties
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added:
oodt/trunk/metadata/src/test/org/apache/oodt/cas/metadata/extractors/TestDataSourceMetExtractor.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/metadata/src/test/org/apache/oodt/cas/metadata/extractors/TestDataSourceMetExtractor.java?rev=1590919&view=auto
==============================================================================
---
oodt/trunk/metadata/src/test/org/apache/oodt/cas/metadata/extractors/TestDataSourceMetExtractor.java
(added)
+++
oodt/trunk/metadata/src/test/org/apache/oodt/cas/metadata/extractors/TestDataSourceMetExtractor.java
Tue Apr 29 08:46:30 2014
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.metadata.extractors;
+
+// JUnit static imports
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Mockito.when;
+
+// JDK imports
+import java.io.File;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.Statement;
+
+// JAVAX imports
+import javax.sql.DataSource;
+
+// OODT imports
+import org.apache.oodt.cas.metadata.Metadata;
+import org.apache.oodt.cas.metadata.exceptions.MetExtractionException;
+
+// JUnit imports
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+// Mockito imports
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
+
+/**
+ * Test class for {@link DataSourceMetExtractor}.
+ *
+ * @author [email protected] (Brian Foster)
+ */
+@RunWith(JUnit4.class)
+public class TestDataSourceMetExtractor {
+
+ private static final String DB_COL_NAME_1 = "NAME";
+ private static final String DB_COL_VALUE_1 = "SomeName";
+ private static final String DB_COL_NAME_2 = "SIZE";
+ private static final String DB_COL_VALUE_2 = "20";
+
+ @Mock private DataSource dataSource;
+ @Mock private Connection conn;
+ @Mock private Statement statement;
+ @Mock private ResultSet rs;
+ @Mock private ResultSetMetaData rsMet;
+
+ private DataSourceMetExtractor extractor;
+
+ @Before
+ public void setUp() throws Exception {
+ MockitoAnnotations.initMocks(this);
+
+ when(dataSource.getConnection()).thenReturn(conn);
+ when(conn.createStatement()).thenReturn(statement);
+ when(statement.executeQuery(Mockito.<String>any())).thenReturn(rs);
+ when(rs.getMetaData()).thenReturn(rsMet);
+ when(rs.getString(0)).thenReturn(DB_COL_VALUE_1);
+ when(rs.getString(1)).thenReturn(DB_COL_VALUE_2);
+ when(rsMet.getColumnCount()).thenReturn(2);
+ when(rsMet.getColumnName(0)).thenReturn(DB_COL_NAME_1);
+ when(rsMet.getColumnName(1)).thenReturn(DB_COL_NAME_2);
+
+ extractor = new DataSourceMetExtractor();
+ }
+
+ @Test
+ public void testGetKey() {
+ assertThat(extractor.getKey(new File("Test.csv")), is("Test"));
+ assertThat(extractor.getKey(new File("123_sdfwegd_g334g.dat")),
is("123_sdfwegd_g334g"));
+ assertThat(extractor.getKey(new File("123qweJDKJF-3")),
is("123qweJDKJF-3"));
+ }
+
+ @Test
+ public void testGetMetadata() throws MetExtractionException {
+ Metadata metadata = extractor.getMetadata(
+ dataSource, "select * from SomeTable where key = '%s'", "SomeKey");
+
+ assertThat(metadata.getAllKeys().size(), is(2));
+ assertThat(metadata.getMetadata(DB_COL_NAME_1), is(DB_COL_VALUE_1));
+ assertThat(metadata.getMetadata(DB_COL_NAME_2), is(DB_COL_VALUE_2));
+ }
+}
Propchange:
oodt/trunk/metadata/src/test/org/apache/oodt/cas/metadata/extractors/TestDataSourceMetExtractor.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
