Repository: falcon Updated Branches: refs/heads/master df92d2b36 -> 2c9295a9d
FALCON-2181 Support for storing metadata of non trusted recipe Author: Praveen Adlakha <[email protected]> Reviewers: @pallavi-rao,@sandeepSamudrala Closes #297 from PraveenAdlakha/751 and squashes the following commits: c8428b7 [Praveen Adlakha] comments addressed fb2c495 [Praveen Adlakha] test cases added and comments addressed f59e622 [Praveen Adlakha] WIP a69d899 [Praveen Adlakha] sandeep's comment addressed d193679 [Praveen Adlakha] FALCON-2181 Support for storing metadata of non trusted recipe Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/2c9295a9 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/2c9295a9 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/2c9295a9 Branch: refs/heads/master Commit: 2c9295a9dac598336688f6389d158f6722d1686f Parents: df92d2b Author: Praveen Adlakha <[email protected]> Authored: Thu Nov 17 15:49:57 2016 +0530 Committer: Pallavi Rao <[email protected]> Committed: Thu Nov 17 15:49:57 2016 +0530 ---------------------------------------------------------------------- .../persistence/ExtensionMetadataBean.java | 112 +++++++++++++++ .../persistence/PersistenceConstants.java | 3 + .../falcon/tools/FalconStateStoreDBCLI.java | 1 + .../src/main/resources/META-INF/persistence.xml | 11 +- extensions/pom.xml | 15 ++ .../apache/falcon/extensions/ExtensionType.java | 37 +++++ .../extensions/jdbc/ExtensionMetaStore.java | 101 ++++++++++++++ .../falcon/extensions/store/ExtensionStore.java | 50 ++++++- .../falcon/extensions/ExtensionServiceTest.java | 4 +- .../extensions/jdbc/ExtensionMetaStoreTest.java | 82 +++++++++++ .../store/AbstractTestExtensionStore.java | 35 +++++ .../test/resources/falcon-buildinfo.properties | 28 ++++ .../src/test/resources/startup.properties | 137 +++++++++++++++++++ .../resource/extensions/ExtensionManager.java | 37 ++--- src/build/findbugs-exclude.xml | 5 + src/main/assemblies/standalone-package.xml | 9 -- .../falcon/resource/ExtensionManagerIT.java | 6 + 17 files changed, 633 insertions(+), 40 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/common/src/main/java/org/apache/falcon/persistence/ExtensionMetadataBean.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/persistence/ExtensionMetadataBean.java b/common/src/main/java/org/apache/falcon/persistence/ExtensionMetadataBean.java new file mode 100644 index 0000000..ac1ded6 --- /dev/null +++ b/common/src/main/java/org/apache/falcon/persistence/ExtensionMetadataBean.java @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.falcon.persistence; + +import javax.persistence.Basic; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.NamedQueries; +import javax.persistence.NamedQuery; +import javax.persistence.Table; +import javax.validation.constraints.NotNull; +import java.util.Date; + + +//SUSPEND CHECKSTYLE CHECK LineLengthCheck +/** + * Table to store extension metadata. + */ + +@Table(name = "EXTENSION_METADATA") +@Entity +@NamedQueries({ + @NamedQuery(name = PersistenceConstants.GET_ALL_EXTENSIONS, query = "select OBJECT(a) from ExtensionMetadataBean a "), + @NamedQuery(name = PersistenceConstants.GET_EXTENSION_LOCATION, query = "select a.location from ExtensionMetadataBean a where a.extensionName = :extensionName"), + @NamedQuery(name = PersistenceConstants.DELETE_EXTENSIONS_OF_TYPE, query = "delete from ExtensionMetadataBean a where a.extensionType = :extensionType ") +}) +//RESUME CHECKSTYLE CHECK LineLengthCheck +public class ExtensionMetadataBean { + @Basic + @NotNull + @Id + @Column(name = "extension_name") + private String extensionName; + + + @Basic + @NotNull + @Column(name = "extension_type") + private String extensionType; + + @Basic + @Column(name = "description") + private String description; + + @Basic + @NotNull + @Column(name = "location") + private String location; + + + @Basic + @NotNull + @Column(name = "creation_time") + private Date creationTime; + + public String getExtensionType() { + return extensionType; + } + + public void setExtensionType(String extensionType) { + this.extensionType = extensionType; + } + + public Date getCreationTime() { + return creationTime; + } + + public void setCreationTime(Date creationTime) { + this.creationTime = creationTime; + } + + + public String getExtensionName() { + return extensionName; + } + + public void setExtensionName(String extensionName) { + this.extensionName = extensionName; + } + + public String getLocation() { + return location; + } + + public void setLocation(String location) { + this.location = location; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } +} http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java b/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java index 8be0eb5..dbbf06d 100644 --- a/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java +++ b/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java @@ -67,4 +67,7 @@ public final class PersistenceConstants { public static final String DELETE_BACKLOG_METRIC_INSTANCE = "DELETE_BACKLOG_METRIC_INSTANCE"; public static final String GET_ALL_BACKLOG_INSTANCES = "GET_ALL_BACKLOG_INSTANCES"; public static final String DELETE_ALL_BACKLOG_ENTITY_INSTANCES ="DELETE_ALL_BACKLOG_ENTITY_INSTANCES"; + public static final String GET_ALL_EXTENSIONS = "GET_ALL_EXTENSIONS"; + public static final String GET_EXTENSION_LOCATION = "GET_EXTENSION_LOCATION"; + public static final String DELETE_EXTENSIONS_OF_TYPE = "DELETE_EXTENSIONS_OF_TYPE"; } http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/common/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java b/common/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java index 9c6e8b3..e12b982 100644 --- a/common/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java +++ b/common/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java @@ -247,6 +247,7 @@ public class FalconStateStoreDBCLI { args.add("org.apache.falcon.persistence.MonitoredEntityBean"); args.add("org.apache.falcon.persistence.EntitySLAAlertBean"); args.add("org.apache.falcon.persistence.BacklogMetricBean"); + args.add("org.apache.falcon.persistence.ExtensionMetadataBean"); return args.toArray(new String[args.size()]); } http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/common/src/main/resources/META-INF/persistence.xml ---------------------------------------------------------------------- diff --git a/common/src/main/resources/META-INF/persistence.xml b/common/src/main/resources/META-INF/persistence.xml index d58e21c..1fbcc9d 100644 --- a/common/src/main/resources/META-INF/persistence.xml +++ b/common/src/main/resources/META-INF/persistence.xml @@ -29,6 +29,7 @@ <class>org.apache.falcon.persistence.MonitoredEntityBean</class> <class>org.apache.falcon.persistence.EntitySLAAlertBean</class> <class>org.apache.falcon.persistence.BacklogMetricBean</class> + <class>org.apache.falcon.persistence.ExtensionMetadataBean</class> <properties> <property name="openjpa.ConnectionDriverName" value="org.apache.commons.dbcp.BasicDataSource"/> @@ -38,7 +39,8 @@ <property name="openjpa.MetaDataFactory" value="jpa(Types=org.apache.falcon.persistence.EntityBean; org.apache.falcon.persistence.InstanceBean;org.apache.falcon.persistence.PendingInstanceBean; - org.apache.falcon.persistence.MonitoredEntityBean;org.apache.falcon.persistence.EntitySLAAlertBean)"></property> + org.apache.falcon.persistence.MonitoredEntityBean;org.apache.falcon.persistence.EntitySLAAlertBean; + org.apache.falcon.persistence.ExtensionMetadataBean)"></property> <property name="openjpa.DetachState" value="fetch-groups(DetachedStateField=true)"/> <property name="openjpa.LockManager" value="pessimistic"/> @@ -62,6 +64,7 @@ <class>org.apache.falcon.persistence.MonitoredEntityBean</class> <class>org.apache.falcon.persistence.EntitySLAAlertBean</class> <class>org.apache.falcon.persistence.BacklogMetricBean</class> + <class>org.apache.falcon.persistence.ExtensionMetadataBean</class> <properties> <property name="openjpa.ConnectionDriverName" value="org.apache.commons.dbcp.BasicDataSource"/> @@ -70,7 +73,8 @@ <property name="openjpa.MetaDataFactory" value="jpa(Types=org.apache.falcon.persistence.EntityBean; org.apache.falcon.persistence.InstanceBean;org.apache.falcon.persistence.PendingInstanceBean; - org.apache.falcon.persistence.MonitoredEntityBean;org.apache.falcon.persistence.EntitySLAAlertBean)"></property> + org.apache.falcon.persistence.MonitoredEntityBean;org.apache.falcon.persistence.EntitySLAAlertBean; + org.apache.falcon.persistence.ExtensionMetadataBean)"></property> <property name="openjpa.DetachState" value="fetch-groups(DetachedStateField=true)"/> <property name="openjpa.LockManager" value="pessimistic"/> @@ -93,6 +97,7 @@ <class>org.apache.falcon.persistence.PendingInstanceBean</class> <class>org.apache.falcon.persistence.EntitySLAAlertBean</class> <class>org.apache.falcon.persistence.BacklogMetricBean</class> + <class>org.apache.falcon.persistence.ExtensionMetadataBean</class> <properties> <property name="openjpa.ConnectionDriverName" value="org.apache.commons.dbcp.BasicDataSource"/> @@ -102,7 +107,7 @@ value="jpa(Types=org.apache.falcon.persistence.EntityBean; org.apache.falcon.persistence.InstanceBean;org.apache.falcon.persistence.PendingInstanceBean; org.apache.falcon.persistence.MonitoredEntityBean;org.apache.falcon.persistence.EntitySLAAlertBean; - org.apache.falcon.persistence.BacklogMetricBean)"/> + org.apache.falcon.persistence.BacklogMetricBean;org.apache.falcon.persistence.ExtensionMetadataBean)"/> <property name="openjpa.DetachState" value="fetch-groups(DetachedStateField=true)"/> <property name="openjpa.LockManager" value="pessimistic"/> <property name="openjpa.ReadLockLevel" value="read"/> http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/extensions/pom.xml ---------------------------------------------------------------------- diff --git a/extensions/pom.xml b/extensions/pom.xml index 34fbad1..77821ca 100644 --- a/extensions/pom.xml +++ b/extensions/pom.xml @@ -103,6 +103,21 @@ <artifactId>falcon-test-util</artifactId> <scope>test</scope> </dependency> + + <dependency> + <groupId>org.apache.derby</groupId> + <artifactId>derby</artifactId> + <version>${derby.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>commons-dbcp</groupId> + <artifactId>commons-dbcp</artifactId> + <version>${commons-dbcp.version}</version> + <scope>test</scope> + </dependency> + </dependencies> <build> http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/extensions/src/main/java/org/apache/falcon/extensions/ExtensionType.java ---------------------------------------------------------------------- diff --git a/extensions/src/main/java/org/apache/falcon/extensions/ExtensionType.java b/extensions/src/main/java/org/apache/falcon/extensions/ExtensionType.java new file mode 100644 index 0000000..c7e45cc --- /dev/null +++ b/extensions/src/main/java/org/apache/falcon/extensions/ExtensionType.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.falcon.extensions; + +/** + * Enum to store ExtensionType. + */ +public enum ExtensionType { + TRUSTED("Trusted extension") , + CUSTOM("Custom extension"); + + private final String text; + + private ExtensionType(final String text) { + this.text = text; + } + @Override + public String toString(){ + return text; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java ---------------------------------------------------------------------- diff --git a/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java b/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java new file mode 100644 index 0000000..b2f7e7d --- /dev/null +++ b/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.falcon.extensions.jdbc; + +import org.apache.falcon.extensions.ExtensionType; +import org.apache.falcon.persistence.ExtensionMetadataBean; +import org.apache.falcon.persistence.PersistenceConstants; +import org.apache.falcon.service.FalconJPAService; + +import javax.persistence.EntityManager; +import javax.persistence.Query; +import java.util.Date; +import java.util.List; + +/** + * Statestore for extension framework. + */ +public class ExtensionMetaStore { + + private EntityManager getEntityManager() { + return FalconJPAService.get().getEntityManager(); + } + + public void storeExtensionMetadataBean(String extensionName, String location, ExtensionType extensionType, + String description){ + ExtensionMetadataBean extensionMetadataBean = new ExtensionMetadataBean(); + extensionMetadataBean.setLocation(location); + extensionMetadataBean.setExtensionName(extensionName); + extensionMetadataBean.setExtensionType(extensionType.toString()); + extensionMetadataBean.setCreationTime(new Date(System.currentTimeMillis())); + extensionMetadataBean.setDescription(description); + EntityManager entityManager = getEntityManager(); + try { + beginTransaction(entityManager); + entityManager.persist(extensionMetadataBean); + } finally { + commitAndCloseTransaction(entityManager); + } + } + + public List<ExtensionMetadataBean> getAllExtensions(){ + EntityManager entityManager = getEntityManager(); + beginTransaction(entityManager); + Query q = entityManager.createNamedQuery(PersistenceConstants.GET_ALL_EXTENSIONS); + try { + return q.getResultList(); + } finally { + commitAndCloseTransaction(entityManager); + } + } + + public void deleteExtensionsOfType(ExtensionType extensionType){ + EntityManager entityManager = getEntityManager(); + beginTransaction(entityManager); + Query q = entityManager.createNamedQuery(PersistenceConstants.DELETE_EXTENSIONS_OF_TYPE); + q.setParameter("extensionType", extensionType.toString()); + try{ + q.executeUpdate(); + } finally { + commitAndCloseTransaction(entityManager); + } + } + + public String getLocation(String extensionName){ + EntityManager entityManager = getEntityManager(); + beginTransaction(entityManager); + Query q = entityManager.createNamedQuery(PersistenceConstants.GET_EXTENSION_LOCATION); + q.setParameter("extensionName", extensionName); + try { + return (String)q.getSingleResult(); + } finally { + commitAndCloseTransaction(entityManager); + } + } + + private void beginTransaction(EntityManager entityManager) { + entityManager.getTransaction().begin(); + } + + private void commitAndCloseTransaction(EntityManager entityManager) { + if (entityManager != null) { + entityManager.getTransaction().commit(); + entityManager.close(); + } + } +} http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java ---------------------------------------------------------------------- diff --git a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java index 074cc73..61804f9 100644 --- a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java +++ b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java @@ -19,6 +19,10 @@ package org.apache.falcon.extensions.store; import org.apache.commons.lang.StringUtils; +import org.apache.falcon.FalconException; +import org.apache.falcon.extensions.AbstractExtension; +import org.apache.falcon.extensions.ExtensionType; +import org.apache.falcon.extensions.jdbc.ExtensionMetaStore; import org.apache.falcon.hadoop.HadoopClientFactory; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -39,6 +43,8 @@ import java.util.List; import java.util.Map; import java.util.HashMap; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,10 +54,19 @@ import org.slf4j.LoggerFactory; public final class ExtensionStore { private static final Logger LOG = LoggerFactory.getLogger(ExtensionStore.class); + + public static ExtensionMetaStore getMetaStore() { + return metaStore; + } + + private static ExtensionMetaStore metaStore = new ExtensionMetaStore(); private FileSystem fs; private Path storePath; + private static final String EXTENSION_PROPERTY_JSON_SUFFIX = "-properties.json"; + private static final String SHORT_DESCRIPTION = "shortDescription"; + // Convention over configuration design paradigm private static final String RESOURCES_DIR = "resources"; private static final String LIBS_DIR = "libs"; @@ -60,6 +75,7 @@ public final class ExtensionStore { private static final ExtensionStore STORE = new ExtensionStore(); + public static ExtensionStore get() { return STORE; } @@ -72,6 +88,39 @@ public final class ExtensionStore { } storePath = new Path(uri); fs = initializeFileSystem(); + initializeDbTable(); + } + + private void initializeDbTable() { + try{ + metaStore.deleteExtensionsOfType(ExtensionType.TRUSTED); + List<String> extensions = getExtensions(); + for (String extension : extensions) { + ExtensionType extensionType = AbstractExtension.isExtensionTrusted(extension) + ? ExtensionType.TRUSTED : ExtensionType.CUSTOM; + String description = getShortDescription(extension); + String recipeName = extension; + String location = storePath.toString() + '/' + extension; + metaStore.storeExtensionMetadataBean(recipeName, location, extensionType, description); + } + } catch (FalconException e){ + LOG.error("Exception in ExtensionStore:", e); + throw new RuntimeException(e); + } + + } + + private String getShortDescription(final String extensionName) throws FalconException { + String content = getResource(extensionName, extensionName.toLowerCase() + + EXTENSION_PROPERTY_JSON_SUFFIX); + String description; + try { + JSONObject jsonObject = new JSONObject(content); + description = (String) jsonObject.get(SHORT_DESCRIPTION); + } catch (JSONException e) { + throw new FalconException(e); + } + return description; } private FileSystem initializeFileSystem() { @@ -83,7 +132,6 @@ public final class ExtensionStore { // set permissions so config store dir is owned by falcon alone HadoopClientFactory.mkdirs(fileSystem, storePath, HadoopClientFactory.ALL_PERMISSION); } - return fileSystem; } catch (Exception e) { throw new RuntimeException("Unable to bring up extension store for path: " + storePath, e); http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/extensions/src/test/java/org/apache/falcon/extensions/ExtensionServiceTest.java ---------------------------------------------------------------------- diff --git a/extensions/src/test/java/org/apache/falcon/extensions/ExtensionServiceTest.java b/extensions/src/test/java/org/apache/falcon/extensions/ExtensionServiceTest.java index c8df2c0..a7c2fda 100644 --- a/extensions/src/test/java/org/apache/falcon/extensions/ExtensionServiceTest.java +++ b/extensions/src/test/java/org/apache/falcon/extensions/ExtensionServiceTest.java @@ -18,6 +18,7 @@ package org.apache.falcon.extensions; +import org.apache.falcon.extensions.store.AbstractTestExtensionStore; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -26,12 +27,13 @@ import org.testng.Assert; /** * Unit tests for ExtensionService. */ -public class ExtensionServiceTest { +public class ExtensionServiceTest extends AbstractTestExtensionStore { private ExtensionService service; @BeforeClass public void setUp() throws Exception { + initExtensionStore(); service = new ExtensionService(); service.init(); } http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java ---------------------------------------------------------------------- diff --git a/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java b/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java new file mode 100644 index 0000000..61b26d6 --- /dev/null +++ b/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java @@ -0,0 +1,82 @@ + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.falcon.extensions.jdbc; + +import org.apache.falcon.cluster.util.EmbeddedCluster; +import org.apache.falcon.extensions.ExtensionType; +import org.apache.falcon.extensions.store.AbstractTestExtensionStore; +import org.apache.falcon.service.FalconJPAService; + +import org.apache.hadoop.conf.Configuration; +import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + + +import javax.persistence.EntityManager; +import javax.persistence.Query; + +/** + * Test Cases for ExtensionMetaStore. + */ + + +public class ExtensionMetaStoreTest extends AbstractTestExtensionStore { + protected EmbeddedCluster dfsCluster; + protected Configuration conf = new Configuration(); + private static ExtensionMetaStore stateStore; + + @BeforeClass + public void setup() throws Exception{ + initExtensionStore(); + this.dfsCluster = EmbeddedCluster.newCluster("testCluster"); + this.conf = dfsCluster.getConf(); + stateStore = new ExtensionMetaStore(); + } + + @BeforeMethod + public void init() { + clear(); + } + + @Test + public void dbOpertaions(){ + //insert + stateStore.storeExtensionMetadataBean("test1", "test_location", ExtensionType.TRUSTED, "test_description"); + Assert.assertEquals(stateStore.getAllExtensions().size(), 1); + //check data + Assert.assertEquals(stateStore.getLocation("test1"), "test_location"); + //delete + stateStore.deleteExtensionsOfType(ExtensionType.TRUSTED); + Assert.assertEquals(stateStore.getAllExtensions().size(), 0); + } + + private void clear() { + EntityManager em = FalconJPAService.get().getEntityManager(); + em.getTransaction().begin(); + try { + Query query = em.createNativeQuery("delete from EXTENSION_METADATA"); + query.executeUpdate(); + } finally { + em.getTransaction().commit(); + em.close(); + } + } +} http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/extensions/src/test/java/org/apache/falcon/extensions/store/AbstractTestExtensionStore.java ---------------------------------------------------------------------- diff --git a/extensions/src/test/java/org/apache/falcon/extensions/store/AbstractTestExtensionStore.java b/extensions/src/test/java/org/apache/falcon/extensions/store/AbstractTestExtensionStore.java index fba803f..cd94eca 100644 --- a/extensions/src/test/java/org/apache/falcon/extensions/store/AbstractTestExtensionStore.java +++ b/extensions/src/test/java/org/apache/falcon/extensions/store/AbstractTestExtensionStore.java @@ -22,10 +22,15 @@ import org.apache.commons.io.FileUtils; import org.apache.falcon.extensions.AbstractExtension; import org.apache.falcon.extensions.ExtensionService; import org.apache.falcon.hadoop.HadoopClientFactory; +import org.apache.falcon.service.FalconJPAService; +import org.apache.falcon.tools.FalconStateStoreDBCLI; import org.apache.falcon.util.StartupProperties; +import org.apache.falcon.util.StateStoreProperties; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; +import org.testng.Assert; import org.testng.annotations.AfterClass; import java.io.File; @@ -41,14 +46,44 @@ public class AbstractTestExtensionStore { protected String extensionStorePath; protected ExtensionStore store; private FileSystem fileSystem; + protected LocalFileSystem fs = new LocalFileSystem(); + private static final String DB_BASE_DIR = "target/test-data/persistancedb"; + protected static final String DB_SQL_FILE = DB_BASE_DIR + File.separator + "out.sql"; + protected static String dbLocation = DB_BASE_DIR + File.separator + "data.db"; + protected static String url = "jdbc:derby:"+ dbLocation +";create=true"; + private static FalconJPAService falconJPAService = FalconJPAService.get(); public void initExtensionStore() throws Exception { initExtensionStore(this.getClass()); } + protected int execDBCLICommands(String[] args) { + return new FalconStateStoreDBCLI().run(args); + } + + public void createDB(String file) { + File sqlFile = new File(file); + String[] argsCreate = { "create", "-sqlfile", sqlFile.getAbsolutePath(), "-run" }; + int result = execDBCLICommands(argsCreate); + Assert.assertEquals(0, result); + Assert.assertTrue(sqlFile.exists()); + + } + public void initExtensionStore(Class resourceClass) throws Exception { + String configPath = new URI(StartupProperties.get().getProperty("config.store.uri")).getPath(); + String location = configPath + "-" + getClass().getName(); + StartupProperties.get().setProperty("config.store.uri", location); + FileUtils.deleteDirectory(new File(location)); + StateStoreProperties.get().setProperty(FalconJPAService.URL, url); + Configuration localConf = new Configuration(); + fs.initialize(LocalFileSystem.getDefaultUri(localConf), localConf); + fs.mkdirs(new Path(DB_BASE_DIR)); + createDB(DB_SQL_FILE); + falconJPAService.init(); new ExtensionService().init(); store = ExtensionService.getExtensionStore(); + FalconJPAService falconJPAService = FalconJPAService.get(); fileSystem = HadoopClientFactory.get().createFalconFileSystem(new Configuration(true)); extensionStorePath = new URI(StartupProperties.get().getProperty(ExtensionStore.EXTENSION_STORE_URI)).getPath(); extensionStoreSetup(resourceClass); http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/extensions/src/test/resources/falcon-buildinfo.properties ---------------------------------------------------------------------- diff --git a/extensions/src/test/resources/falcon-buildinfo.properties b/extensions/src/test/resources/falcon-buildinfo.properties new file mode 100644 index 0000000..5a7cb82 --- /dev/null +++ b/extensions/src/test/resources/falcon-buildinfo.properties @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +###################### +*.domain=all + +*.build.user=${user.name} +*.build.epoch=${timestamp} +*.project.version=${pom.version} +*.build.version=${pom.version}-r${buildNumber} +*.vc.revision=${buildNumber} +*.vc.source.url=${scm.connection} +###################### http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/extensions/src/test/resources/startup.properties ---------------------------------------------------------------------- diff --git a/extensions/src/test/resources/startup.properties b/extensions/src/test/resources/startup.properties new file mode 100644 index 0000000..58daf32 --- /dev/null +++ b/extensions/src/test/resources/startup.properties @@ -0,0 +1,137 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +*.domain=debug + +######### Implementation classes ######### +## DONT MODIFY UNLESS SURE ABOUT CHANGE ## + +*.workflow.engine.impl=org.apache.falcon.workflow.engine.OozieWorkflowEngine +*.oozie.process.workflow.builder=org.apache.falcon.workflow.OozieProcessWorkflowBuilder +*.oozie.feed.workflow.builder=org.apache.falcon.workflow.OozieFeedWorkflowBuilder +*.SchedulableEntityManager.impl=org.apache.falcon.resource.SchedulableEntityManager +*.ConfigSyncService.impl=org.apache.falcon.resource.ConfigSyncService +*.ProcessInstanceManager.impl=org.apache.falcon.resource.InstanceManager +*.catalog.service.impl=org.apache.falcon.catalog.HiveCatalogService + +##### Falcon Services ##### +*.application.services=org.apache.falcon.security.AuthenticationInitializationService,\ + org.apache.falcon.workflow.WorkflowJobEndNotificationService, \ + org.apache.falcon.service.ProcessSubscriberService,\ + org.apache.falcon.entity.store.ConfigurationStore,\ + org.apache.falcon.rerun.service.RetryService,\ + org.apache.falcon.rerun.service.LateRunService,\ + org.apache.falcon.notification.service.impl.JobCompletionService,\ + org.apache.falcon.notification.service.impl.SchedulerService,\ + org.apache.falcon.notification.service.impl.AlarmService,\ + org.apache.falcon.notification.service.impl.DataAvailabilityService,\ + org.apache.falcon.execution.FalconExecutionService,\ + org.apache.falcon.service.FalconJPAService + +##### Falcon Configuration Store Change listeners ##### +*.configstore.listeners=org.apache.falcon.entity.v0.EntityGraph,\ + org.apache.falcon.entity.ColoClusterRelation,\ + org.apache.falcon.group.FeedGroupMap,\ + org.apache.falcon.entity.store.FeedLocationStore + +##### JMS MQ Broker Implementation class ##### +*.broker.impl.class=org.apache.activemq.ActiveMQConnectionFactory + + +######### System startup parameters ######### + +# Location to store user entity configurations +debug.config.store.uri=file://${user.dir}/target/store +debug.config.store.persist=false +debug.config.oozie.conf.uri=${user.dir}/target/oozie +debug.system.lib.location=${system.lib.location} +debug.broker.url=vm://localhost +debug.retry.recorder.path=${user.dir}/target/retry +debug.libext.feed.retention.paths=${falcon.libext} +debug.libext.feed.replication.paths=${falcon.libext} +debug.libext.process.paths=${falcon.libext} + +*.falcon.cleanup.service.frequency=minutes(5) + +*.extension.store.uri=file://${user.dir}/target/extensions + + +######### Properties for configuring JMS provider - activemq ######### +# Default Active MQ url +*.broker.url=tcp://localhost:61616 + +# default time-to-live for a JMS message 3 days (time in minutes) +*.broker.ttlInMins=4320 +*.entity.topic=FALCON.ENTITY.TOPIC +*.max.retry.failure.count=1 +*.retry.recorder.path=${user.dir}/logs/retry + +######### Properties for configuring iMon client and metric ######### +*.internal.queue.size=1000 + + +##### List of shared libraries for Falcon workflows ##### +*.shared.libs=activemq-all,ant,geronimo-j2ee-management,jms,json-simple,oozie-client,spring-jms,commons-lang3,commons-el + +######### Authentication Properties ######### + +# Authentication type must be specified: simple|kerberos +*.falcon.authentication.type=simple + +##### Service Configuration + +# Indicates the Kerberos principal to be used in Falcon Service. +*.falcon.service.authentication.kerberos.principal= + +# Location of the keytab file with the credentials for the Service principal. +*.falcon.service.authentication.kerberos.keytab= + +# name node principal to talk to config store +*.dfs.namenode.kerberos.principal= + +##### SPNEGO Configuration + +# Authentication type must be specified: simple|kerberos|<class> +# org.apache.falcon.security.RemoteUserInHeaderBasedAuthenticationHandler can be used for backwards compatibility +*.falcon.http.authentication.type=simple + +# Indicates how long (in seconds) an authentication token is valid before it has to be renewed. +*.falcon.http.authentication.token.validity=36000 + +# The signature secret for signing the authentication tokens. +*.falcon.http.authentication.signature.secret=falcon + +# The domain to use for the HTTP cookie that stores the authentication token. +*.falcon.http.authentication.cookie.domain= + +# Indicates if anonymous requests are allowed when using 'simple' authentication. +*.falcon.http.authentication.simple.anonymous.allowed=false + +# Indicates the Kerberos principal to be used for HTTP endpoint. +# The principal MUST +#rt with 'HTTP/' as per Kerberos HTTP SPNEGO specification. +*.falcon.http.authentication.kerberos.principal= + +# Location of the keytab file with the credentials for the HTTP principal. +*.falcon.http.authentication.kerberos.keytab= + +# The kerberos names rules is to resolve kerberos principal names, refer to Hadoop's KerberosName for more details. +*.falcon.http.authentication.kerberos.name.rules=DEFAULT + +# Comma separated list of black listed users +*.falcon.http.authentication.blacklisted.users= http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java ---------------------------------------------------------------------- diff --git a/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java b/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java index 92b5531..138109c 100644 --- a/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java +++ b/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java @@ -26,11 +26,12 @@ import org.apache.falcon.entity.v0.Entity; import org.apache.falcon.entity.v0.cluster.Cluster; import org.apache.falcon.entity.v0.feed.Feed; import org.apache.falcon.entity.v0.process.Process; -import org.apache.falcon.extensions.AbstractExtension; import org.apache.falcon.extensions.Extension; import org.apache.falcon.extensions.ExtensionProperties; import org.apache.falcon.extensions.ExtensionService; +import org.apache.falcon.extensions.jdbc.ExtensionMetaStore; import org.apache.falcon.extensions.store.ExtensionStore; +import org.apache.falcon.persistence.ExtensionMetadataBean; import org.apache.falcon.resource.APIResult; import org.apache.falcon.resource.AbstractSchedulableEntityManager; import org.apache.falcon.resource.EntityList; @@ -84,15 +85,12 @@ public class ExtensionManager extends AbstractSchedulableEntityManager { private static final String EXTENSION_RESULTS = "extensions"; private static final String TOTAL_RESULTS = "totalResults"; private static final String README = "README"; - private static final String EXTENSION_PROPERTY_JSON_SUFFIX = "-properties.json"; - private static final String SHORT_DESCRIPTION = "shortDescription"; private static final String EXTENSION_NAME = "name"; private static final String EXTENSION_TYPE = "type"; private static final String EXTENSION_DESC = "description"; + public static final String EXTENSION_LOCATION = "location"; - private static final String TRUSTED_EXTENSION = "Trusted extension"; - private static final String CUSTOM_EXTENSION = "Custom extension"; - + private static final String EXTENSION_PROPERTY_JSON_SUFFIX = "-properties.json"; //SUSPEND CHECKSTYLE CHECK ParameterNumberCheck @GET @Path("list/{extension-name}") @@ -446,17 +444,16 @@ public class ExtensionManager extends AbstractSchedulableEntityManager { private static JSONArray buildEnumerateResult(final List<String> extensions) throws FalconException { JSONArray results = new JSONArray(); - - for (String extension : extensions) { - String extensionType = AbstractExtension.isExtensionTrusted(extension) ? TRUSTED_EXTENSION - : CUSTOM_EXTENSION; - + ExtensionMetaStore metricStore = ExtensionStore.get().getMetaStore(); + List<ExtensionMetadataBean> beanList = metricStore.getAllExtensions(); + for (ExtensionMetadataBean bean : beanList) { JSONObject resultObject = new JSONObject(); try { - resultObject.put(EXTENSION_NAME, extension.toLowerCase()); - resultObject.put(EXTENSION_TYPE, extensionType); - resultObject.put(EXTENSION_DESC, getShortDescription(extension)); + resultObject.put(EXTENSION_NAME, bean.getExtensionName().toLowerCase()); + resultObject.put(EXTENSION_TYPE, bean.getExtensionType()); + resultObject.put(EXTENSION_DESC, bean.getDescription()); + resultObject.put(EXTENSION_LOCATION, bean.getLocation()); } catch (JSONException e) { throw new FalconException(e); } @@ -466,18 +463,6 @@ public class ExtensionManager extends AbstractSchedulableEntityManager { return results; } - private static String getShortDescription(final String extensionName) throws FalconException { - String content = ExtensionStore.get().getResource(extensionName, extensionName.toLowerCase() - + EXTENSION_PROPERTY_JSON_SUFFIX); - String description; - try { - JSONObject jsonObject = new JSONObject(content); - description = (String) jsonObject.get(SHORT_DESCRIPTION); - } catch (JSONException e) { - throw new FalconException(e); - } - return description; - } private List<Entity> generateEntities(String extensionName, HttpServletRequest request) throws FalconException, IOException { http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/src/build/findbugs-exclude.xml ---------------------------------------------------------------------- diff --git a/src/build/findbugs-exclude.xml b/src/build/findbugs-exclude.xml index 346583d..d910c96 100644 --- a/src/build/findbugs-exclude.xml +++ b/src/build/findbugs-exclude.xml @@ -70,6 +70,11 @@ </Match> <Match> + <Class name="org.apache.falcon.persistence.ExtensionMetadataBean" /> + <Bug pattern="NP_BOOLEAN_RETURN_NULL,UWF_UNWRITTEN_FIELD" /> + </Match> + + <Match> <Bug pattern="RV_RETURN_VALUE_IGNORED_NO_SIDE_EFFECT" /> </Match> </FindBugsFilter> http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/src/main/assemblies/standalone-package.xml ---------------------------------------------------------------------- diff --git a/src/main/assemblies/standalone-package.xml b/src/main/assemblies/standalone-package.xml index c3b2259..26df008 100644 --- a/src/main/assemblies/standalone-package.xml +++ b/src/main/assemblies/standalone-package.xml @@ -271,15 +271,6 @@ <directoryMode>0755</directoryMode> </fileSet> - <fileSet> - <directory>../</directory> - <outputDirectory>extensions/mirroring</outputDirectory> - <excludes> - <exclude>*/**</exclude> - </excludes> - <fileMode>0770</fileMode> - <directoryMode>0770</directoryMode> - </fileSet> </fileSets> <files> http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/webapp/src/test/java/org/apache/falcon/resource/ExtensionManagerIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/falcon/resource/ExtensionManagerIT.java b/webapp/src/test/java/org/apache/falcon/resource/ExtensionManagerIT.java index db23d6d..349cfec 100644 --- a/webapp/src/test/java/org/apache/falcon/resource/ExtensionManagerIT.java +++ b/webapp/src/test/java/org/apache/falcon/resource/ExtensionManagerIT.java @@ -158,6 +158,12 @@ public class ExtensionManagerIT extends AbstractTestExtensionStore { System.out.println("extension -submitAndSchedule -extensionName hdfs-mirroring"); Assert.assertEquals(TestContext.executeWithURL( "extension -submitAndSchedule -extensionName hdfs-mirroring"), -1); + + // enumerate + + System.out.println("extension -enumerate"); + Assert.assertEquals(TestContext.executeWithURL( + "extension -enumerate"), 3); } private Map<String, String> getHDFSMirroringProperty(String jobName, String start, String end) {
