Repository: falcon Updated Branches: refs/heads/master 3059980c6 -> 4445798be
FALCON-1658 MySql Support for Native Scheduler. Contributed by Pavan Kumar Kolamuri. Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/4445798b Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/4445798b Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/4445798b Branch: refs/heads/master Commit: 4445798bebd0420d86b5f091354b1155a3a09f77 Parents: 3059980 Author: Ajay Yadava <[email protected]> Authored: Mon Dec 14 19:53:02 2015 +0530 Committer: Ajay Yadava <[email protected]> Committed: Mon Dec 14 19:55:04 2015 +0530 ---------------------------------------------------------------------- CHANGES.txt | 2 ++ pom.xml | 1 + scheduler/pom.xml | 13 ++++++++-- .../falcon/tools/FalconStateStoreDBCLI.java | 2 +- .../src/main/resources/META-INF/persistence.xml | 27 ++++++++++++++++++++ src/bin/falcon-db.sh | 2 +- src/conf/startup.properties | 25 ++++++++++++++---- 7 files changed, 63 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/4445798b/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 4d42881..b594160 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -31,6 +31,8 @@ Trunk (Unreleased) FALCON-1213 Base framework of the native scheduler(Pallavi Rao) IMPROVEMENTS + FALCON-1658 MySql Support for Native Scheduler(Pavan Kumar Kolamuri via Ajay Yadava) + FALCON-1656 Improve FeedHelper:getRetentionFrequency method(Ajay Yadava) FALCON-1616 Consume Workflow job end notifications for SLA monitoring(Sandeep Samudrala via Ajay Yadava) http://git-wip-us.apache.org/repos/asf/falcon/blob/4445798b/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 3ad8789..a7b21ae 100644 --- a/pom.xml +++ b/pom.xml @@ -117,6 +117,7 @@ <javax-validation.version>1.0.0.GA</javax-validation.version> <derby.version>10.10.1.1</derby.version> <commons-dbcp.version>1.4</commons-dbcp.version> + <mysql.version>5.1.36</mysql.version> <internal.maven.repo>file:///tmp/falcontemprepo</internal.maven.repo> <excluded.test.groups>exhaustive</excluded.test.groups> </properties> http://git-wip-us.apache.org/repos/asf/falcon/blob/4445798b/scheduler/pom.xml ---------------------------------------------------------------------- diff --git a/scheduler/pom.xml b/scheduler/pom.xml index c305651..c934b3e 100644 --- a/scheduler/pom.xml +++ b/scheduler/pom.xml @@ -135,7 +135,13 @@ <dependency> <groupId>org.apache.derby</groupId> <artifactId>derby</artifactId> - <version>10.10.1.1</version> + <version>${derby.version}</version> + </dependency> + + <dependency> + <groupId>mysql</groupId> + <artifactId>mysql-connector-java</artifactId> + <version>${mysql.version}</version> </dependency> <dependency> @@ -196,7 +202,10 @@ <artifactItem> <groupId>commons-dbcp</groupId> <artifactId>commons-dbcp</artifactId> - <version>${commons-dbcp.version}</version> + </artifactItem> + <artifactItem> + <groupId>mysql</groupId> + <artifactId>mysql-connector-java</artifactId> </artifactItem> </artifactItems> <outputDirectory>${project.build.directory}/dependency</outputDirectory> http://git-wip-us.apache.org/repos/asf/falcon/blob/4445798b/scheduler/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java ---------------------------------------------------------------------- diff --git a/scheduler/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java b/scheduler/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java index 381b0b3..f4058c8 100644 --- a/scheduler/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java +++ b/scheduler/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java @@ -51,7 +51,7 @@ public class FalconStateStoreDBCLI { // Represents whether DB instance exists or not. private boolean instanceExists; - private static final String[] FALCON_HELP = {"Falcon DB initialization tool currently supports Derby DB"}; + private static final String[] FALCON_HELP = {"Falcon DB initialization tool currently supports Derby DB/ Mysql"}; public static void main(String[] args) { new FalconStateStoreDBCLI().run(args); http://git-wip-us.apache.org/repos/asf/falcon/blob/4445798b/scheduler/src/main/resources/META-INF/persistence.xml ---------------------------------------------------------------------- diff --git a/scheduler/src/main/resources/META-INF/persistence.xml b/scheduler/src/main/resources/META-INF/persistence.xml index 86558de..99e116c 100644 --- a/scheduler/src/main/resources/META-INF/persistence.xml +++ b/scheduler/src/main/resources/META-INF/persistence.xml @@ -47,4 +47,31 @@ </properties> </persistence-unit> + <persistence-unit name="falcon-mysql" transaction-type="RESOURCE_LOCAL"> + <provider>org.apache.openjpa.persistence.PersistenceProviderImpl</provider> + + <class>org.apache.falcon.state.store.jdbc.EntityBean</class> + <class>org.apache.falcon.state.store.jdbc.InstanceBean</class> + + <properties> + <property name="openjpa.ConnectionDriverName" value="org.apache.commons.dbcp.BasicDataSource"/> + + <property name="openjpa.ConnectionProperties" value="**INVALID**"/> <!--Set by StoreService at init time --> + + <property name="openjpa.MetaDataFactory" + value="jpa(Types=org.apache.falcon.state.store.EntityBean; + org.apache.falcon.state.store.InstanceBean)"></property> + + <property name="openjpa.DetachState" value="fetch-groups(DetachedStateField=true)"/> + <property name="openjpa.LockManager" value="pessimistic"/> + <property name="openjpa.ReadLockLevel" value="read"/> + <property name="openjpa.WriteLockLevel" value="write"/> + <property name="openjpa.jdbc.TransactionIsolation" value="repeatable-read"/> <!--CUSTOM--> + <property name="openjpa.jdbc.DBDictionary" value="batchLimit=50"/> + <property name="openjpa.jdbc.DBDictionary" value="TimestampTypeName=TIMESTAMP"/> + <property name="openjpa.RuntimeUnenhancedClasses" value="unsupported"/> + <property name="openjpa.Log" value="log4j"/> + </properties> + </persistence-unit> + </persistence> http://git-wip-us.apache.org/repos/asf/falcon/blob/4445798b/src/bin/falcon-db.sh ---------------------------------------------------------------------- diff --git a/src/bin/falcon-db.sh b/src/bin/falcon-db.sh index 415fd5d..bdeaf8c 100644 --- a/src/bin/falcon-db.sh +++ b/src/bin/falcon-db.sh @@ -46,4 +46,4 @@ done # set the client class path FALCONCPPATH="$FALCONCPPATH:${BASEDIR}/client/lib/*" -${JAVA_BIN} ${JAVA_PROPERTIES} -cp ${FALCONCPPATH} org.apache.falcon.tools.FalconStateStoreDBCLI "${@}" +${JAVA_BIN} ${JAVA_PROPERTIES} -cp ${FALCONCPPATH} -Dfalcon.log.dir=$HOME org.apache.falcon.tools.FalconStateStoreDBCLI "${@}" http://git-wip-us.apache.org/repos/asf/falcon/blob/4445798b/src/conf/startup.properties ---------------------------------------------------------------------- diff --git a/src/conf/startup.properties b/src/conf/startup.properties index 1694930..95d792b 100644 --- a/src/conf/startup.properties +++ b/src/conf/startup.properties @@ -27,7 +27,6 @@ ## If you wish to use Falcon native scheduler uncomment and enable the properties below.## #*.dag.engine.impl=org.apache.falcon.workflow.engine.OozieDAGEngine -#*.state.store.impl=org.apache.falcon.state.store.jdbc.JdbcStateStore ## If you wish to use Falcon native scheduler set the workflow engine to FalconWorkflowEngine instead of OozieWorkflowEngine. ## #*.workflow.engine.impl=org.apache.falcon.workflow.engine.FalconWorkflowEngine *.workflow.engine.impl=org.apache.falcon.workflow.engine.OozieWorkflowEngine @@ -52,13 +51,28 @@ org.apache.falcon.service.LogCleanupService,\ org.apache.falcon.service.GroupsService,\ org.apache.falcon.service.ProxyUserService -## If you wish to use Falcon native scheduler add the commented out services below to application.services ## +## If you wish to use Falcon native scheduler uncomment out below application services and +# comment out above application +# +#services ## +#*.application.services=org.apache.falcon.security.AuthenticationInitializationService,\ +# org.apache.falcon.workflow.WorkflowJobEndNotificationService, \ +# org.apache.falcon.service.ProcessSubscriberService,\ +# org.apache.falcon.service.FeedSLAMonitoringService,\ +# org.apache.falcon.service.LifecyclePolicyMap,\ +# org.apache.falcon.state.store.service.FalconJPAService,\ +# org.apache.falcon.entity.store.ConfigurationStore,\ +# org.apache.falcon.rerun.service.RetryService,\ +# org.apache.falcon.rerun.service.LateRunService,\ +# org.apache.falcon.metadata.MetadataMappingService,\ +# org.apache.falcon.service.LogCleanupService,\ +# org.apache.falcon.service.GroupsService,\ +# org.apache.falcon.service.ProxyUserService,\ # org.apache.falcon.notification.service.impl.JobCompletionService,\ # org.apache.falcon.notification.service.impl.SchedulerService,\ # org.apache.falcon.notification.service.impl.AlarmService,\ # org.apache.falcon.notification.service.impl.DataAvailabilityService,\ -# org.apache.falcon.execution.FalconExecutionService,\ -# org.apache.falcon.state.store.service.FalconJPAService +# org.apache.falcon.execution.FalconExecutionService ##### Prism Services ##### prism.application.services=org.apache.falcon.service.LifecyclePolicyMap,\ @@ -78,7 +92,7 @@ prism.application.services=org.apache.falcon.service.LifecyclePolicyMap,\ org.apache.falcon.service.FeedSLAMonitoringService,\ org.apache.falcon.service.SharedLibraryHostingService ## If you wish to use Falcon native scheduler, add the State store as a configstore listener. ## -# org.apache.falcon.state.store.jdbc.JdbcStateStore +# org.apache.falcon.state.store.jdbc.JDBCStateStore ##### Prism Configuration Store Change listeners ##### prism.configstore.listeners=org.apache.falcon.entity.v0.EntityGraph,\ @@ -267,6 +281,7 @@ prism.configstore.listeners=org.apache.falcon.entity.v0.EntityGraph,\ ######### StateStore Properties ##### #*.falcon.state.store.impl=org.apache.falcon.state.store.jdbc.JDBCStateStore #*.falcon.statestore.jdbc.driver=org.apache.derby.jdbc.EmbeddedDriver +## Falcon currently supports derby and mysql, change url based on DB. #*.falcon.statestore.jdbc.url=jdbc:derby:data/statestore.db;create=true #*.falcon.statestore.jdbc.username=sa #*.falcon.statestore.jdbc.password=
