Repository: falcon Updated Branches: refs/heads/master 093ad2797 -> 5ff837819
FALCON-1664 Add Postgres support for native scheduler.Contributed by Deepak Barr. Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/5ff83781 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/5ff83781 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/5ff83781 Branch: refs/heads/master Commit: 5ff837819f5a6f31070d5f8bfbecacdd1eeb0ab9 Parents: 093ad27 Author: Pragya <[email protected]> Authored: Wed Jan 27 10:31:35 2016 +0000 Committer: Pragya <[email protected]> Committed: Wed Jan 27 10:31:35 2016 +0000 ---------------------------------------------------------------------- CHANGES.txt | 3 +++ common/src/main/resources/statestore.properties | 2 +- docs/src/site/twiki/FalconNativeScheduler.twiki | 2 +- pom.xml | 1 + scheduler/pom.xml | 10 ++++++++ .../falcon/state/store/jdbc/InstanceBean.java | 4 +-- .../falcon/tools/FalconStateStoreDBCLI.java | 3 ++- .../src/main/resources/META-INF/persistence.xml | 27 ++++++++++++++++++++ src/conf/statestore.properties | 2 +- 9 files changed, 48 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/5ff83781/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index d87ce26..90b32ca 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -3,6 +3,9 @@ Trunk TASKS: INCOMPATIBLE CHANGES NEW FEATURES + + FALCON-1664 Add Postgres support for native scheduler(Deepak Barr via Pragya Mittal) + FALCON-1495 In instance status list, show all runs for instances when requested by user(Narayan Periwal via Ajay Yadava) FALCON-1230 Data based notification Service to notify execution instances when data becomes available(Pavan Kumar Kolamuri via Ajay Yadava) http://git-wip-us.apache.org/repos/asf/falcon/blob/5ff83781/common/src/main/resources/statestore.properties ---------------------------------------------------------------------- diff --git a/common/src/main/resources/statestore.properties b/common/src/main/resources/statestore.properties index a67a871..44e79b3 100644 --- a/common/src/main/resources/statestore.properties +++ b/common/src/main/resources/statestore.properties @@ -21,7 +21,7 @@ ######### StateStore Properties ##### #*.falcon.state.store.impl=org.apache.falcon.state.store.jdbc.JDBCStateStore #*.falcon.statestore.jdbc.driver=org.apache.derby.jdbc.EmbeddedDriver -## Falcon currently supports derby and mysql, change url based on DB. +## Falcon currently supports derby, mysql and postgreSQL, change url based on DB. #*.falcon.statestore.jdbc.url=jdbc:derby:data/falcon.db;create=true ## StateStore credentials file where username,password and other properties can be stored securely. http://git-wip-us.apache.org/repos/asf/falcon/blob/5ff83781/docs/src/site/twiki/FalconNativeScheduler.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/FalconNativeScheduler.twiki b/docs/src/site/twiki/FalconNativeScheduler.twiki index 6f0679e..de6bfe7 100644 --- a/docs/src/site/twiki/FalconNativeScheduler.twiki +++ b/docs/src/site/twiki/FalconNativeScheduler.twiki @@ -113,7 +113,7 @@ usage: schema </verbatim> -Currently, MySQL and Derby are supported as state stores. We may extend support to other DBs in the future. Falcon has been tested against MySQL v5.5. +Currently, MySQL, postgreSQL and Derby are supported as state stores. We may extend support to other DBs in the future. Falcon has been tested against MySQL v5.5 and PostgreSQL v9.5. ---++++ Using Derby as the State Store Using Derby is ideal for QA and staging setup. Falcon comes bundled with a Derby connector and no explicit setup is required (although you can set it up) in terms creating the DB or tables. http://git-wip-us.apache.org/repos/asf/falcon/blob/5ff83781/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index a80822f..94f7aee 100644 --- a/pom.xml +++ b/pom.xml @@ -110,6 +110,7 @@ <derby.version>10.10.1.1</derby.version> <commons-dbcp.version>1.4</commons-dbcp.version> <mysql.version>5.1.36</mysql.version> + <postgres.version>9.1-901-1.jdbc4</postgres.version> <internal.maven.repo>file:///tmp/falcontemprepo</internal.maven.repo> <excluded.test.groups>exhaustive</excluded.test.groups> </properties> http://git-wip-us.apache.org/repos/asf/falcon/blob/5ff83781/scheduler/pom.xml ---------------------------------------------------------------------- diff --git a/scheduler/pom.xml b/scheduler/pom.xml index 6392e98..2e2b2e9 100644 --- a/scheduler/pom.xml +++ b/scheduler/pom.xml @@ -145,6 +145,12 @@ </dependency> <dependency> + <groupId>postgresql</groupId> + <artifactId>postgresql</artifactId> + <version>${postgres.version}</version> + </dependency> + + <dependency> <groupId>commons-dbcp</groupId> <artifactId>commons-dbcp</artifactId> <version>${commons-dbcp.version}</version> @@ -207,6 +213,10 @@ <groupId>mysql</groupId> <artifactId>mysql-connector-java</artifactId> </artifactItem> + <artifactItem> + <groupId>postgresql</groupId> + <artifactId>postgresql</artifactId> + </artifactItem> </artifactItems> <outputDirectory>${project.build.directory}/dependency</outputDirectory> </configuration> http://git-wip-us.apache.org/repos/asf/falcon/blob/5ff83781/scheduler/src/main/java/org/apache/falcon/state/store/jdbc/InstanceBean.java ---------------------------------------------------------------------- diff --git a/scheduler/src/main/java/org/apache/falcon/state/store/jdbc/InstanceBean.java b/scheduler/src/main/java/org/apache/falcon/state/store/jdbc/InstanceBean.java index e8385b1..dffb116 100644 --- a/scheduler/src/main/java/org/apache/falcon/state/store/jdbc/InstanceBean.java +++ b/scheduler/src/main/java/org/apache/falcon/state/store/jdbc/InstanceBean.java @@ -114,11 +114,11 @@ public class InstanceBean { private EntityBean entityBean; - @Column(name = "awaited_predicates", columnDefinition = "BLOB") + @Column(name = "awaited_predicates") @Lob private byte[] awaitedPredicates; - @Column(name = "properties", columnDefinition = "BLOB") + @Column(name = "properties") @Lob private byte[] properties; http://git-wip-us.apache.org/repos/asf/falcon/blob/5ff83781/scheduler/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java ---------------------------------------------------------------------- diff --git a/scheduler/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java b/scheduler/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java index 7f22b0e..6de9f7d 100644 --- a/scheduler/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java +++ b/scheduler/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java @@ -51,7 +51,8 @@ public class FalconStateStoreDBCLI { // Represents whether DB instance exists or not. private boolean instanceExists; - private static final String[] FALCON_HELP = {"Falcon DB initialization tool currently supports Derby DB/ Mysql"}; + private static final String[] FALCON_HELP = + {"Falcon DB initialization tool currently supports Derby DB/ Mysql/ PostgreSQL"}; public static void main(String[] args) { new FalconStateStoreDBCLI().run(args); http://git-wip-us.apache.org/repos/asf/falcon/blob/5ff83781/scheduler/src/main/resources/META-INF/persistence.xml ---------------------------------------------------------------------- diff --git a/scheduler/src/main/resources/META-INF/persistence.xml b/scheduler/src/main/resources/META-INF/persistence.xml index 99e116c..c2ef794 100644 --- a/scheduler/src/main/resources/META-INF/persistence.xml +++ b/scheduler/src/main/resources/META-INF/persistence.xml @@ -74,4 +74,31 @@ </properties> </persistence-unit> + <persistence-unit name="falcon-postgresql" transaction-type="RESOURCE_LOCAL"> + <provider>org.apache.openjpa.persistence.PersistenceProviderImpl</provider> + + <class>org.apache.falcon.state.store.jdbc.EntityBean</class> + <class>org.apache.falcon.state.store.jdbc.InstanceBean</class> + + <properties> + <property name="openjpa.ConnectionDriverName" value="org.apache.commons.dbcp.BasicDataSource"/> + + <property name="openjpa.ConnectionProperties" value="**INVALID**"/> <!--Set by StoreService at init time --> + + <property name="openjpa.MetaDataFactory" + value="jpa(Types=org.apache.falcon.state.store.EntityBean; + org.apache.falcon.state.store.InstanceBean)"></property> + + <property name="openjpa.DetachState" value="fetch-groups(DetachedStateField=true)"/> + <property name="openjpa.LockManager" value="pessimistic"/> + <property name="openjpa.ReadLockLevel" value="read"/> + <property name="openjpa.WriteLockLevel" value="write"/> + <property name="openjpa.jdbc.TransactionIsolation" value="repeatable-read"/> <!--CUSTOM--> + <property name="openjpa.jdbc.DBDictionary" value="batchLimit=50"/> + <property name="openjpa.jdbc.DBDictionary" value="TimestampTypeName=TIMESTAMP"/> + <property name="openjpa.RuntimeUnenhancedClasses" value="unsupported"/> + <property name="openjpa.Log" value="log4j"/> + </properties> + </persistence-unit> + </persistence> http://git-wip-us.apache.org/repos/asf/falcon/blob/5ff83781/src/conf/statestore.properties ---------------------------------------------------------------------- diff --git a/src/conf/statestore.properties b/src/conf/statestore.properties index de5ad8f..5659531 100644 --- a/src/conf/statestore.properties +++ b/src/conf/statestore.properties @@ -21,7 +21,7 @@ ######### StateStore Properties ##### #*.falcon.state.store.impl=org.apache.falcon.state.store.jdbc.JDBCStateStore #*.falcon.statestore.jdbc.driver=org.apache.derby.jdbc.EmbeddedDriver -## Falcon currently supports derby and mysql, change url based on DB. +## Falcon currently supports derby, mysql and postgreSQL, change url based on DB. #*.falcon.statestore.jdbc.url=jdbc:derby:data/falcon.db;create=true ## StateStore credentials file where username,password and other properties can be stored securely.
