FALCON-2066 Extend support for Import and Export of various database types This is a replacement for pull request - https://github.com/apache/falcon/pull/224.
+ Checkstyle violations addressed + Removed the changes that went into pull-218 + Created https://issues.apache.org/jira/browse/FALCON-2083 to track the direct mode & verbose + Removed tab from the xml files Author: Venkatesan Ramachandran <[email protected]> Reviewers: "Peeyush B <[email protected]>, Balu Vellanki <[email protected]>" Closes #229 from vramachan/FALCON-2066.DatabaseTypes Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/292dfed3 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/292dfed3 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/292dfed3 Branch: refs/heads/master Commit: 292dfed341b462488ab58584dc305f519f1beb4d Parents: 9a4c18c Author: Venkatesan Ramachandran <[email protected]> Authored: Tue Jul 19 11:43:50 2016 -0700 Committer: bvellanki <[email protected]> Committed: Tue Jul 19 11:43:50 2016 -0700 ---------------------------------------------------------------------- client/src/main/resources/datasource-0.1.xsd | 5 ++ docs/src/site/twiki/EntitySpecification.twiki | 8 ++ examples/entity/importexport/README.txt | 56 ++++++++++++ examples/entity/importexport/db2-datasource.xml | 56 ++++++++++++ .../importexport/db2-feed-import-filesystem.xml | 71 ++++++++++++++++ .../importexport/feed-export-filesystem.xml | 63 ++++++++++++++ .../entity/importexport/feed-export-hcat.xml | 65 ++++++++++++++ .../feed-export-repl-filesystem.xml | 72 ++++++++++++++++ .../feed-import-export-filesystem.xml | 80 ++++++++++++++++++ .../feed-import-export-repl-filesystem.xml | 89 ++++++++++++++++++++ .../entity/importexport/feed-import-hcat.xml | 69 +++++++++++++++ .../feed-import-repl-filesystem.xml | 81 ++++++++++++++++++ .../entity/importexport/generic-datasource.xml | 49 +++++++++++ .../generic-feed-import-filesystem.xml | 72 ++++++++++++++++ .../entity/importexport/mysql-datasource.xml | 58 +++++++++++++ .../mysql-feed-export-filesystem.xml | 62 ++++++++++++++ .../mysql-feed-import-filesystem.xml | 72 ++++++++++++++++ .../entity/importexport/netezza-datasource.xml | 57 +++++++++++++ .../netezza-feed-import-filesystem.xml | 72 ++++++++++++++++ .../entity/importexport/oracle-datasource.xml | 50 +++++++++++ .../oracle-feed-import-filesystem.xml | 72 ++++++++++++++++ .../entity/importexport/pgsql-datasource.xml | 55 ++++++++++++ .../pgsql-feed-import-filesystem.xml | 70 +++++++++++++++ .../entity/importexport/primary_cluster.xml | 33 ++++++++ examples/entity/importexport/td-datasource.xml | 56 ++++++++++++ .../importexport/td-feed-import-filesystem.xml | 70 +++++++++++++++ .../oozie/DatabaseExportWorkflowBuilder.java | 28 ++---- .../oozie/DatabaseImportWorkflowBuilder.java | 37 ++------ .../apache/falcon/oozie/ImportExportCommon.java | 22 ++++- .../OozieOrchestrationWorkflowBuilder.java | 21 +---- 30 files changed, 1603 insertions(+), 68 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/client/src/main/resources/datasource-0.1.xsd ---------------------------------------------------------------------- diff --git a/client/src/main/resources/datasource-0.1.xsd b/client/src/main/resources/datasource-0.1.xsd index ef78239..d78d967 100644 --- a/client/src/main/resources/datasource-0.1.xsd +++ b/client/src/main/resources/datasource-0.1.xsd @@ -232,6 +232,11 @@ <xs:enumeration value="mysql"/> <xs:enumeration value="oracle"/> <xs:enumeration value="hsql"/> + <xs:enumeration value="postgres"/> + <xs:enumeration value="db2"/> + <xs:enumeration value="netezza"/> + <xs:enumeration value="teradata"/> + <xs:enumeration value="generic"/> </xs:restriction> </xs:simpleType> http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/docs/src/site/twiki/EntitySpecification.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/EntitySpecification.twiki b/docs/src/site/twiki/EntitySpecification.twiki index faad305..2615e5d 100644 --- a/docs/src/site/twiki/EntitySpecification.twiki +++ b/docs/src/site/twiki/EntitySpecification.twiki @@ -117,6 +117,14 @@ Following are the tags defined in a datasource.xml: name="test-hsql-db" xmlns="uri:falcon:datasource:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> </verbatim> +---+++ Datasource Types + +Falcon currently supports relational databases as data sources (both source and target). The following relational +databases are supported + * MySQL, HSQL, Postgres, Oracle, Teradata, Netezza, DB2 + * Generic - generic jdbc data source. This will require specifying a driver classname and jar file in the + datasource entity specification. Please see samples in the examples dir. + The colo specifies the colo to which the datasource belongs to and name is the name of the datasource which has to be unique. http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/README.txt ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/README.txt b/examples/entity/importexport/README.txt new file mode 100755 index 0000000..8b20aa5 --- /dev/null +++ b/examples/entity/importexport/README.txt @@ -0,0 +1,56 @@ +####### Using MySQL as Datasource ####### +yum install mysql mysql-server mysql-libs +/usr/share/java/mysql-connector-java.jar +cp /home/ambari-qa/falcon_ingestion/backup/mysql-connector-java-5.1.31.jar /usr/share/java/ +unlink /usr/share/java/mysql-connector-java.jar +ln -s /home/ambari-qa/falcon_ingestion/backup/mysql-connector-java-5.1.31.jar /usr/share/java/mysql-connector-java.jar +service mysqld start + +####### oozie sharelib change for sqoop ###################### +# Copy /usr/share/java/mysql-connector-java.jar (version 5.1.31) into oozie share lib of sqoop dir +hadoop dfs -put /usr/share/java/mysql-connector-java-5.1.31.jar /user/oozie/share/lib/lib_XXXXXXXXXXX/sqoop + +####### Setup MySQL user and create source table ############## +# Setup - Create MySQL source table - db_raw_data +############################################################### +use test; +create user sqoop_user@localhost identified by 'sqoop'; +grant all privileges on *.* to 'sqoop_user'@'localhost' with grant option; +create user 'sqoop_user'@'%' identified by 'sqoop'; +grant all privileges on *.* to 'sqoop_user'@'%' with grant option; +flush privileges; + +## Import: following is the data source tables from which data will be pulled into HADOOP + +create table db_raw_data(id int not null, name varchar(20), value int, modified_ts timestamp); +insert into db_raw_data values (1, 'Apple', 700, now() ); +insert into db_raw_data values (2, 'Blackberry', 1, now() ); +insert into db_raw_data values (3, 'Cisco', 100, now() ); +insert into db_raw_data values (4, 'Delta', 10, now() ); +insert into db_raw_data values (5, 'Eagle', 55, now() ); +insert into db_raw_data values (6, 'Falcon', 99, now() ); + +## submit the following falcon entities and schedule + +falcon entity -type cluster -file primary_cluster.xml -submit +falcon entity -type datasource -file mysql-datasource.xml -submit +falcon entity -type feed -file mysql-feed-import-filesystem.xml -submitAndSchedule + +## Export: create a target table in the MySQL database + +create table db_export_fs(id int not null, name varchar(20), value int, modified_ts timestamp); + +## submit the following falcon entities and schedule +falcon entity -type feed -file mysql-feed-export-filesystem.xml -submitAndSchedule + +### Note +# For other supported database types like Oracle, Postgres, Teradata, Netezza, DB2, Generic JDB +# please use the corresponding feed entity that starts with the database name in the xml entity +# definition. + +# Also, please make sure to copy the jdbc driver onto the Oozie Sqoop shared lib on HDFS and +# restart Oozie. The datasource entity definition will also refer to the JDBC drivers in order +# for falcon to validate the connection when submitting the datasource entity. + + + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/db2-datasource.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/db2-datasource.xml b/examples/entity/importexport/db2-datasource.xml new file mode 100755 index 0000000..36a061b --- /dev/null +++ b/examples/entity/importexport/db2-datasource.xml @@ -0,0 +1,56 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<datasource colo="west-coast" description="DB2 database on west coast" type="db2" name="db2-db" xmlns="uri:falcon:datasource:0.1"> + <tags>[email protected], [email protected]</tags> + <interfaces> + + <!-- ***** read interface ***** --> + <interface type="readonly" endpoint="jdbc:db2://172.25.16.65:50000/Sample"> + <credential type="password-text"> + <userName>sqoop</userName> + <passwordText>sqoop</passwordText> + </credential> + </interface> + + <!-- ***** write interface ***** --> + <interface type="write" endpoint="jdbc:db2://172.25.16.65:50000/Sample"> + <credential type="password-text"> + <userName>sqoop</userName> + <passwordText>sqoop</passwordText> + </credential> + </interface> + + <!-- ***** default credential ***** --> + <credential type="password-alias"> + <userName>sqoop</userName> + <passwordAlias> + <alias>sqoop.password.alias</alias> + <providerPath>hdfs://c6401.ambari.apache.org:8020/user/ambari-qa/sqoop_password.jceks</providerPath> + </passwordAlias> + </credential> + + </interfaces> + + <driver> + <clazz>com.ibm.db2.jcc.DB2Driver</clazz> + <jar>/user/oozie/share/lib/lib_20160628052542/sqoop/db2jcc4.jar</jar> + </driver> +</datasource> + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/db2-feed-import-filesystem.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/db2-feed-import-filesystem.xml b/examples/entity/importexport/db2-feed-import-filesystem.xml new file mode 100755 index 0000000..2adca37 --- /dev/null +++ b/examples/entity/importexport/db2-feed-import-filesystem.xml @@ -0,0 +1,71 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<!-- + A feed representing Hourly customer email data retained for 90 days + --> +<feed description="Raw customer email feed" name="db2-feed-import-filesystem" xmlns="uri:falcon:feed:0.1"> + <tags>externalSystem=USWestEmailServers,classification=secure</tags> + <groups>DataImportPipeline</groups> + <frequency>minutes(10)</frequency> + <late-arrival cut-off="hours(4)"/> + <clusters> + <cluster name="primaryCluster" type="source"> + <validity start="2016-04-10T00:00Z" end="2017-03-31T00:00Z"/> + <retention limit="days(90)" action="delete"/> + <import> + <source name="db2-db" tableName="db_raw_data"> + <extract type="full"> + <mergepolicy>snapshot</mergepolicy> + </extract> + <fields> + <includes> + <field>id</field> + <field>name</field> + </includes> + </fields> + </source> + <arguments> + <argument name="--split-by" value="id"/> + <argument name="--num-mappers" value="2"/> + </arguments> + </import> + </cluster> + </clusters> + + <locations> + <location type="data" path="/user/ambari-qa/falcon/demo/primary/db2-feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/> + <location type="stats" path="/none"/> + <location type="meta" path="/none"/> + </locations> + + <ACL owner="ambari-qa" group="users" permission="0755"/> + <schema location="/none" provider="none"/> + + <lifecycle> + <retention-stage> + <frequency>days(1)</frequency> + <queue>default</queue> + <priority>LOW</priority> + <properties> + <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property> + </properties> + </retention-stage> + </lifecycle> +</feed> http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/feed-export-filesystem.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/feed-export-filesystem.xml b/examples/entity/importexport/feed-export-filesystem.xml new file mode 100755 index 0000000..36d2c79 --- /dev/null +++ b/examples/entity/importexport/feed-export-filesystem.xml @@ -0,0 +1,63 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<!-- + A feed representing Hourly customer email data retained for 90 days + --> +<feed description="Raw customer email feed" name="feed-export-filesystem" xmlns="uri:falcon:feed:0.1"> + <tags>externalSystem=USWestEmailServers,classification=secure</tags> + <groups>DataImportPipeline</groups> + <frequency>minutes(10)</frequency> + <late-arrival cut-off="hours(4)"/> + <clusters> + <cluster name="primaryCluster" type="source"> + <validity start="2016-01-01T00:00Z" end="2017-03-31T00:00Z"/> + <retention limit="days(90)" action="delete"/> + <export> + <target name="mysql-db" tableName="simple_export_filesystem"> + <load type="allowinsert"/> + </target> + <arguments> + <argument name="--update-key" value="id"/> + </arguments> + </export> + </cluster> + </clusters> + + <locations> + <location type="data" path="/user/ambari-qa/falcon/demo/primary/mysql-feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/> + <location type="stats" path="/none"/> + <location type="meta" path="/none"/> + </locations> + + <ACL owner="ambari-qa" group="users" permission="0755"/> + <schema location="/none" provider="none"/> + + <lifecycle> + <retention-stage> + <frequency>days(1)</frequency> + <queue>default</queue> + <priority>LOW</priority> + <properties> + <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property> + </properties> + </retention-stage> + </lifecycle> +</feed> http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/feed-export-hcat.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/feed-export-hcat.xml b/examples/entity/importexport/feed-export-hcat.xml new file mode 100755 index 0000000..f52bf20 --- /dev/null +++ b/examples/entity/importexport/feed-export-hcat.xml @@ -0,0 +1,65 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<!-- + A feed representing Hourly customer email data retained for 90 days + --> +<feed description="Raw customer email feed" name="feed-export-hcat" xmlns="uri:falcon:feed:0.1"> + <tags>externalSystem=USWestEmailServers,classification=secure</tags> + <groups>DataImportPipeline</groups> + <frequency>minutes(10)</frequency> + <late-arrival cut-off="hours(4)"/> + <clusters> + <cluster name="primaryCluster" type="source"> + <validity start="2016-01-01T00:00Z" end="2016-03-31T00:00Z"/> + <retention limit="days(90)" action="delete"/> + <export> + <target name="mysql-db" tableName="db_processed_data"> + <load type="allowinsert"/> + <!--fields> + <includes> + <field>id</field> + <field>name</field> + </includes> + </fields--> + </target> + <arguments> + <argument name="--update-key" value="id"/> + </arguments> + </export> + </cluster> + </clusters> + + <table uri="catalog:default:hcat_raw_data#year=${YEAR};month=${MONTH};day=${DAY};hour=${HOUR};minute=${MINUTE}"/> + + <ACL owner="ambari-qa" group="users" permission="0755"/> + <schema location="/none" provider="none"/> + + <lifecycle> + <retention-stage> + <frequency>days(1)</frequency> + <queue>default</queue> + <priority>LOW</priority> + <properties> + <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property> + </properties> + </retention-stage> + </lifecycle> +</feed> http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/feed-export-repl-filesystem.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/feed-export-repl-filesystem.xml b/examples/entity/importexport/feed-export-repl-filesystem.xml new file mode 100755 index 0000000..1932b5b --- /dev/null +++ b/examples/entity/importexport/feed-export-repl-filesystem.xml @@ -0,0 +1,72 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<!-- + A feed representing Hourly customer email data retained for 90 days + --> +<feed description="Raw customer email feed" name="feed-export-repl-filesystem" xmlns="uri:falcon:feed:0.1"> + <tags>externalSystem=USWestEmailServers,classification=secure</tags> + <groups>DataImportPipeline</groups> + <frequency>minutes(10)</frequency> + <late-arrival cut-off="hours(4)"/> + <clusters> + <cluster name="primaryCluster" type="source"> + <validity start="2016-01-01T00:00Z" end="2016-03-31T00:00Z"/> + <retention limit="days(90)" action="delete"/> + <export> + <target name="mysql-db" tableName="simple_export_filesystem"> + <load type="allowinsert"/> + </target> + <arguments> + <argument name="--update-key" value="id"/> + </arguments> + </export> + </cluster> + <cluster name="backupCluster" type="target"> + <validity start="2016-01-01T00:00Z" end="2017-03-31T00:00Z"/> + <retention limit="days(90)" action="delete"/> + <locations> + <location type="data" path="/user/ambari-qa/falcon/demo/backup/feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/> + <location type="stats" path="/none"/> + <location type="meta" path="/none"/> + </locations> + </cluster> + </clusters> + + <locations> + <location type="data" path="/user/ambari-qa/falcon/demo/primary/feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/> + <location type="stats" path="/none"/> + <location type="meta" path="/none"/> + </locations> + + <ACL owner="ambari-qa" group="users" permission="0755"/> + <schema location="/none" provider="none"/> + + <lifecycle> + <retention-stage> + <frequency>days(1)</frequency> + <queue>default</queue> + <priority>LOW</priority> + <properties> + <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property> + </properties> + </retention-stage> + </lifecycle> +</feed> http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/feed-import-export-filesystem.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/feed-import-export-filesystem.xml b/examples/entity/importexport/feed-import-export-filesystem.xml new file mode 100755 index 0000000..4e9e215 --- /dev/null +++ b/examples/entity/importexport/feed-import-export-filesystem.xml @@ -0,0 +1,80 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<!-- + A feed representing Hourly customer email data retained for 90 days + --> +<feed description="Raw customer email feed" name="feed-import-export-filesystem" xmlns="uri:falcon:feed:0.1"> + <tags>externalSystem=USWestEmailServers,classification=secure</tags> + <groups>DataImportPipeline</groups> + <frequency>minutes(10)</frequency> + <late-arrival cut-off="hours(4)"/> + <clusters> + <cluster name="primaryCluster" type="source"> + <validity start="2016-01-01T00:00Z" end="2016-03-31T00:00Z"/> + <retention limit="days(90)" action="delete"/> + <import> + <source name="mysql-db" tableName="simple"> + <extract type="full"> + <mergepolicy>snapshot</mergepolicy> + </extract> + <fields> + <includes> + <field>id</field> + <field>name</field> + </includes> + </fields> + </source> + <arguments> + <argument name="--split-by" value="id"/> + <argument name="--num-mappers" value="2"/> + </arguments> + </import> + <export> + <target name="mysql-db" tableName="simple_export"> + <load type="allowinsert"/> + </target> + <arguments> + <argument name="--update-key" value="id"/> + </arguments> + </export> + </cluster> + </clusters> + + <locations> + <location type="data" path="/user/ambari-qa/falcon/demo/primary/feed-import-export/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/> + <location type="stats" path="/none"/> + <location type="meta" path="/none"/> + </locations> + + <ACL owner="ambari-qa" group="users" permission="0755"/> + <schema location="/none" provider="none"/> + + <lifecycle> + <retention-stage> + <frequency>days(1)</frequency> + <queue>default</queue> + <priority>LOW</priority> + <properties> + <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property> + </properties> + </retention-stage> + </lifecycle> +</feed> http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/feed-import-export-repl-filesystem.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/feed-import-export-repl-filesystem.xml b/examples/entity/importexport/feed-import-export-repl-filesystem.xml new file mode 100755 index 0000000..4e9a9be --- /dev/null +++ b/examples/entity/importexport/feed-import-export-repl-filesystem.xml @@ -0,0 +1,89 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<!-- + A feed representing Hourly customer email data retained for 90 days + --> +<feed description="Raw customer email feed" name="feed-import-export-repl-filesystem" xmlns="uri:falcon:feed:0.1"> + <tags>externalSystem=USWestEmailServers,classification=secure</tags> + <groups>DataImportPipeline</groups> + <frequency>minutes(10)</frequency> + <late-arrival cut-off="hours(4)"/> + <clusters> + <cluster name="primaryCluster" type="source"> + <validity start="2016-01-01T00:00Z" end="2016-03-31T00:00Z"/> + <retention limit="days(90)" action="delete"/> + <import> + <source name="mysql-db" tableName="simple"> + <extract type="full"> + <mergepolicy>snapshot</mergepolicy> + </extract> + <fields> + <includes> + <field>id</field> + <field>name</field> + </includes> + </fields> + </source> + <arguments> + <argument name="--split-by" value="id"/> + <argument name="--num-mappers" value="2"/> + </arguments> + </import> + <export> + <target name="mysql-db" tableName="simple_import_export_filesystem"> + <load type="allowinsert"/> + </target> + <arguments> + <argument name="--update-key" value="id"/> + </arguments> + </export> + </cluster> + <cluster name="backupCluster" type="target"> + <validity start="2016-01-01T00:00Z" end="2017-03-31T00:00Z"/> + <retention limit="days(90)" action="delete"/> + <locations> + <location type="data" path="/user/ambari-qa/falcon/demo/backup/feed-import-export-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/> + <location type="stats" path="/none"/> + <location type="meta" path="/none"/> + </locations> + </cluster> + </clusters> + + <locations> + <location type="data" path="/user/ambari-qa/falcon/demo/primary/feed-import-export-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/> + <location type="stats" path="/none"/> + <location type="meta" path="/none"/> + </locations> + + <ACL owner="ambari-qa" group="users" permission="0755"/> + <schema location="/none" provider="none"/> + + <lifecycle> + <retention-stage> + <frequency>days(1)</frequency> + <queue>default</queue> + <priority>LOW</priority> + <properties> + <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property> + </properties> + </retention-stage> + </lifecycle> +</feed> http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/feed-import-hcat.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/feed-import-hcat.xml b/examples/entity/importexport/feed-import-hcat.xml new file mode 100755 index 0000000..73d2bf4 --- /dev/null +++ b/examples/entity/importexport/feed-import-hcat.xml @@ -0,0 +1,69 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<!-- + A feed representing Hourly customer email data retained for 90 days + --> +<feed description="Raw customer email feed" name="feed-import-hcat" xmlns="uri:falcon:feed:0.1"> + <tags>externalSystem=USWestEmailServers,classification=secure</tags> + <groups>DataImportPipeline</groups> + <frequency>minutes(10)</frequency> + <late-arrival cut-off="hours(4)"/> + <clusters> + <cluster name="primaryCluster" type="source"> + <validity start="2016-01-01T00:00Z" end="2016-03-31T00:00Z"/> + <retention limit="days(90)" action="delete"/> + <import> + <source name="mysql-db" tableName="mysql_raw_data"> + <extract type="full"> + <mergepolicy>snapshot</mergepolicy> + </extract> + <fields> + <includes> + <field>id</field> + <field>name</field> + </includes> + </fields> + </source> + <arguments> + <argument name="--split-by" value="id"/> + <argument name="--num-mappers" value="2"/> + </arguments> + </import> + </cluster> + </clusters> + + <!--<table uri="catalog:default:hcat_raw_data#year=${YEAR};month=${MONTH};day=${DAY};hour=${HOUR};minute=${MINUTE}"/>--> + <table uri="catalog:default:hcat_raw_data#year=BLAH"/> + + <ACL owner="ambari-qa" group="users" permission="0755"/> + <schema location="/none" provider="none"/> + + <lifecycle> + <retention-stage> + <frequency>days(1)</frequency> + <queue>default</queue> + <priority>LOW</priority> + <properties> + <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property> + </properties> + </retention-stage> + </lifecycle> +</feed> http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/feed-import-repl-filesystem.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/feed-import-repl-filesystem.xml b/examples/entity/importexport/feed-import-repl-filesystem.xml new file mode 100755 index 0000000..2bc110a --- /dev/null +++ b/examples/entity/importexport/feed-import-repl-filesystem.xml @@ -0,0 +1,81 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<!-- + A feed representing Hourly customer email data retained for 90 days + --> +<feed description="Raw customer email feed" name="feed-import-repl-filesystem" xmlns="uri:falcon:feed:0.1"> + <tags>externalSystem=USWestEmailServers,classification=secure</tags> + <groups>DataImportPipeline</groups> + <frequency>minutes(10)</frequency> + <late-arrival cut-off="hours(4)"/> + <clusters> + <cluster name="primaryCluster" type="source"> + <validity start="2016-01-01T00:00Z" end="2017-03-31T00:00Z"/> + <retention limit="days(90)" action="delete"/> + <import> + <source name="mysql-db" tableName="mysql_raw_data"> + <extract type="full"> + <mergepolicy>snapshot</mergepolicy> + </extract> + <fields> + <includes> + <field>id</field> + <field>name</field> + </includes> + </fields> + </source> + <arguments> + <argument name="--split-by" value="id"/> + <argument name="--num-mappers" value="2"/> + </arguments> + </import> + </cluster> + <cluster name="backupCluster" type="target"> + <validity start="2016-01-01T00:00Z" end="2017-03-31T00:00Z"/> + <retention limit="days(90)" action="delete"/> + <locations> + <location type="data" path="/user/ambari-qa/falcon/demo/backup/feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/> + <location type="stats" path="/none"/> + <location type="meta" path="/none"/> + </locations> + </cluster> + </clusters> + + <locations> + <location type="data" path="/user/ambari-qa/falcon/demo/primary/feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/> + <location type="stats" path="/none"/> + <location type="meta" path="/none"/> + </locations> + + <ACL owner="ambari-qa" group="users" permission="0755"/> + <schema location="/none" provider="none"/> + + <lifecycle> + <retention-stage> + <frequency>days(1)</frequency> + <queue>default</queue> + <priority>LOW</priority> + <properties> + <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property> + </properties> + </retention-stage> + </lifecycle> +</feed> http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/generic-datasource.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/generic-datasource.xml b/examples/entity/importexport/generic-datasource.xml new file mode 100755 index 0000000..74f15ca --- /dev/null +++ b/examples/entity/importexport/generic-datasource.xml @@ -0,0 +1,49 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<datasource colo="west-coast" description="Generic database on west coast" type="generic" name="generic-db" xmlns="uri:falcon:datasource:0.1"> + <tags>[email protected], [email protected]</tags> + <interfaces> + + <!-- ***** read interface ***** --> + <interface type="readonly" endpoint="jdbc:mysql://c6402/test"> + <credential type="password-text"> + <userName>sqoop_user</userName> + <passwordText>sqoop</passwordText> + </credential> + </interface> + + <!-- ***** write interface ***** --> + <interface type="write" endpoint="jdbc:mysql://c6402/test"> + <credential type="password-text"> + <userName>sqoop2_user</userName> + <passwordText>sqoop</passwordText> + </credential> + </interface> + + </interfaces> + + <driver> + <clazz>com.mysql.jdbc.Driver</clazz> + <jar>/user/oozie/share/lib/lib_20160628052542/sqoop/mysql-connector-java-5.1.31.jar</jar> + </driver> + +</datasource> + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/generic-feed-import-filesystem.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/generic-feed-import-filesystem.xml b/examples/entity/importexport/generic-feed-import-filesystem.xml new file mode 100755 index 0000000..01ccdab --- /dev/null +++ b/examples/entity/importexport/generic-feed-import-filesystem.xml @@ -0,0 +1,72 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<!-- + A feed representing Hourly customer email data retained for 90 days + --> +<feed description="Raw customer email feed" name="generic-feed-import-filesystem" xmlns="uri:falcon:feed:0.1"> + <tags>externalSystem=USWestEmailServers,classification=secure</tags> + <groups>DataImportPipeline</groups> + <frequency>minutes(10)</frequency> + <late-arrival cut-off="hours(4)"/> + <clusters> + <cluster name="primaryCluster" type="source"> + <validity start="2016-04-10T00:00Z" end="2017-03-31T00:00Z"/> + <retention limit="days(90)" action="delete"/> + <import> + <source name="generic-db" tableName="db_raw_data"> + <extract type="full"> + <mergepolicy>snapshot</mergepolicy> + </extract> + <fields> + <includes> + <field>id</field> + <field>name</field> + </includes> + </fields> + </source> + <arguments> + <argument name="--split-by" value="id"/> + <argument name="--num-mappers" value="2"/> + </arguments> + </import> + </cluster> + </clusters> + + <locations> + <location type="data" path="/user/ambari-qa/falcon/demo/primary/generic-feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/> + <location type="stats" path="/none"/> + <location type="meta" path="/none"/> + </locations> + + <ACL owner="ambari-qa" group="users" permission="0755"/> + <schema location="/none" provider="none"/> + + <lifecycle> + <retention-stage> + <frequency>days(1)</frequency> + <queue>default</queue> + <priority>LOW</priority> + <properties> + <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property> + </properties> + </retention-stage> + </lifecycle> +</feed> http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/mysql-datasource.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/mysql-datasource.xml b/examples/entity/importexport/mysql-datasource.xml new file mode 100755 index 0000000..293b0bb --- /dev/null +++ b/examples/entity/importexport/mysql-datasource.xml @@ -0,0 +1,58 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<datasource colo="west-coast" description="MySQL database on west coast" type="mysql" name="mysql-db" xmlns="uri:falcon:datasource:0.1"> + <tags>[email protected], [email protected]</tags> + <interfaces> + + <!-- ***** read interface ***** --> + <interface type="readonly" endpoint="jdbc:mysql://c6402/test"> + <credential type="password-text"> + <userName>sqoop_user</userName> + <passwordText>sqoop</passwordText> + </credential> + </interface> + + <!-- ***** write interface ***** --> + <interface type="write" endpoint="jdbc:mysql://c6402/test"> + <credential type="password-text"> + <userName>sqoop2_user</userName> + <passwordText>sqoop</passwordText> + </credential> + </interface> + + <!-- ***** default credential ***** --> + <credential type="password-alias"> + <userName>sqoop2_user</userName> + <passwordAlias> + <alias>sqoop.password.alias</alias> + <providerPath>hdfs://c6407.ambari.apache.org:8020/user/ambari-qa/sqoop_password.jceks</providerPath> + </passwordAlias> + </credential> + + </interfaces> + + <driver> + <clazz>com.mysql.jdbc.Driver</clazz> + <jar>/user/oozie/share/lib/lib_20160628052542/sqoop/mysql-connector-java-5.1.31.jar</jar> + </driver> + +</datasource> + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/mysql-feed-export-filesystem.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/mysql-feed-export-filesystem.xml b/examples/entity/importexport/mysql-feed-export-filesystem.xml new file mode 100755 index 0000000..2e07aa2 --- /dev/null +++ b/examples/entity/importexport/mysql-feed-export-filesystem.xml @@ -0,0 +1,62 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --># + +<!-- + A feed representing Hourly customer email data retained for 90 days + --> +<feed description="Raw customer email feed" name="mysql-feed-export-filesystem" xmlns="uri:falcon:feed:0.1"> + <tags>externalSystem=USWestEmailServers,classification=secure</tags> + <groups>DataImportPipeline</groups> + <frequency>minutes(10)</frequency> + <late-arrival cut-off="hours(4)"/> + <clusters> + <cluster name="primaryCluster" type="source"> + <validity start="2016-04-10T00:00Z" end="2017-03-31T00:00Z"/> + <retention limit="days(90)" action="delete"/> + <export> + <target name="mysql-db" tableName="db_export_fs"> + <load type="allowinsert"/> + </target> + <arguments> + <argument name="--update-key" value="id"/> + </arguments> + </export> + </cluster> + </clusters> + + <locations> + <location type="data" path="/user/ambari-qa/falcon/demo/primary/mysql-feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/> + <location type="stats" path="/none"/> + <location type="meta" path="/none"/> + </locations> + + <ACL owner="ambari-qa" group="users" permission="0755"/> + <schema location="/none" provider="none"/> + + <lifecycle> + <retention-stage> + <frequency>days(1)</frequency> + <queue>default</queue> + <priority>LOW</priority> + <properties> + <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property> + </properties> + </retention-stage> + </lifecycle> +</feed> http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/mysql-feed-import-filesystem.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/mysql-feed-import-filesystem.xml b/examples/entity/importexport/mysql-feed-import-filesystem.xml new file mode 100755 index 0000000..b02b86f --- /dev/null +++ b/examples/entity/importexport/mysql-feed-import-filesystem.xml @@ -0,0 +1,72 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<!-- + A feed representing Hourly customer email data retained for 90 days + --> +<feed description="Raw customer email feed" name="mysql-feed-import-filesystem" xmlns="uri:falcon:feed:0.1"> + <tags>externalSystem=USWestEmailServers,classification=secure</tags> + <groups>DataImportPipeline</groups> + <frequency>minutes(10)</frequency> + <late-arrival cut-off="hours(4)"/> + <clusters> + <cluster name="primaryCluster" type="source"> + <validity start="2016-04-10T00:00Z" end="2017-03-31T00:00Z"/> + <retention limit="days(90)" action="delete"/> + <import> + <source name="mysql-db" tableName="db_raw_data"> + <extract type="full"> + <mergepolicy>snapshot</mergepolicy> + </extract> + <fields> + <includes> + <field>id</field> + <field>name</field> + </includes> + </fields> + </source> + <arguments> + <argument name="--split-by" value="id"/> + <argument name="--num-mappers" value="2"/> + </arguments> + </import> + </cluster> + </clusters> + + <locations> + <location type="data" path="/user/ambari-qa/falcon/demo/primary/mysql-feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/> + <location type="stats" path="/none"/> + <location type="meta" path="/none"/> + </locations> + + <ACL owner="ambari-qa" group="users" permission="0755"/> + <schema location="/none" provider="none"/> + + <lifecycle> + <retention-stage> + <frequency>days(1)</frequency> + <queue>default</queue> + <priority>LOW</priority> + <properties> + <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property> + </properties> + </retention-stage> + </lifecycle> +</feed> http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/netezza-datasource.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/netezza-datasource.xml b/examples/entity/importexport/netezza-datasource.xml new file mode 100755 index 0000000..1e4dfe6 --- /dev/null +++ b/examples/entity/importexport/netezza-datasource.xml @@ -0,0 +1,57 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<datasource colo="west-coast" description="Netezza database on west coast" type="netezza" name="netezza-db" xmlns="uri:falcon:datasource:0.1"> + <tags>[email protected], [email protected]</tags> + <interfaces> + + <!-- ***** read interface ***** --> + <interface type="readonly" endpoint="jdbc:netezza://10.10.20.14:5480/test"> + <credential type="password-text"> + <userName>admin</userName> + <passwordText>password</passwordText> + </credential> + </interface> + + <!-- ***** write interface ***** --> + <interface type="write" endpoint="jdbc:netezza://10.10.20.14:5480/test"> + <credential type="password-text"> + <userName>admin</userName> + <passwordText>password</passwordText> + </credential> + </interface> + + <!-- ***** default credential ***** --> + <credential type="password-alias"> + <userName>admin</userName> + <passwordAlias> + <alias>sqoop.password.alias</alias> + <providerPath>hdfs://c6401.ambari.apache.org:8020/user/ambari-qa/sqoop_password.jceks</providerPath> + </passwordAlias> + </credential> + + </interfaces> + + <driver> + <clazz>org.netezza.Driver</clazz> + <jar>/user/oozie/share/lib/lib_20160628052542/sqoop/nzjdbc.jar</jar> + </driver> +</datasource> + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/netezza-feed-import-filesystem.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/netezza-feed-import-filesystem.xml b/examples/entity/importexport/netezza-feed-import-filesystem.xml new file mode 100755 index 0000000..bbbcbad --- /dev/null +++ b/examples/entity/importexport/netezza-feed-import-filesystem.xml @@ -0,0 +1,72 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<!-- + A feed representing Hourly customer email data retained for 90 days + --> +<feed description="Raw customer email feed" name="netezza-feed-import-filesystem" xmlns="uri:falcon:feed:0.1"> + <tags>externalSystem=USWestEmailServers,classification=secure</tags> + <groups>DataImportPipeline</groups> + <frequency>minutes(10)</frequency> + <late-arrival cut-off="hours(4)"/> + <clusters> + <cluster name="primaryCluster" type="source"> + <validity start="2016-04-10T00:00Z" end="2017-03-31T00:00Z"/> + <retention limit="days(90)" action="delete"/> + <import> + <source name="netezza-db" tableName="db_raw_data"> + <extract type="full"> + <mergepolicy>snapshot</mergepolicy> + </extract> + <fields> + <includes> + <field>id</field> + <field>name</field> + </includes> + </fields> + </source> + <arguments> + <argument name="--split-by" value="id"/> + <argument name="--num-mappers" value="2"/> + </arguments> + </import> + </cluster> + </clusters> + + <locations> + <location type="data" path="/user/ambari-qa/falcon/demo/primary/netezza-feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/> + <location type="stats" path="/none"/> + <location type="meta" path="/none"/> + </locations> + + <ACL owner="ambari-qa" group="users" permission="0755"/> + <schema location="/none" provider="none"/> + + <lifecycle> + <retention-stage> + <frequency>days(1)</frequency> + <queue>default</queue> + <priority>LOW</priority> + <properties> + <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property> + </properties> + </retention-stage> + </lifecycle> +</feed> http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/oracle-datasource.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/oracle-datasource.xml b/examples/entity/importexport/oracle-datasource.xml new file mode 100755 index 0000000..535640c --- /dev/null +++ b/examples/entity/importexport/oracle-datasource.xml @@ -0,0 +1,50 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<datasource colo="west-coast" description="Oracle database on west coast" type="oracle" name="oracle-db" xmlns="uri:falcon:datasource:0.1"> + <tags>[email protected], [email protected]</tags> + <interfaces> + + <!-- ***** read interface ***** --> + <interface type="readonly" endpoint="jdbc:oracle:thin:@172.25.16.82:1521:XE"> + <credential type="password-text"> + <userName>SQOOP</userName> + <passwordText>sqoop</passwordText> + </credential> + </interface> + + <!-- ***** write interface ***** --> + <interface type="write" endpoint="jdbc:oracle:thin:@172.25.16.82:1521:XE"> + <credential type="password-text"> + <userName>SQOOP</userName> + <passwordText>sqoop</passwordText> + </credential> + </interface> + + </interfaces> + + <driver> + <clazz>oracle.jdbc.OracleDriver</clazz> + <jar>/user/oozie/share/lib/lib_20160628052542/sqoop/ojdbc6.jar</jar> + <jar>/user/oozie/share/lib/lib_20160628052542/sqoop/orai18n.jar</jar> + </driver> + +</datasource> + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/oracle-feed-import-filesystem.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/oracle-feed-import-filesystem.xml b/examples/entity/importexport/oracle-feed-import-filesystem.xml new file mode 100755 index 0000000..5bba3de --- /dev/null +++ b/examples/entity/importexport/oracle-feed-import-filesystem.xml @@ -0,0 +1,72 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<!-- + A feed representing Hourly customer email data retained for 90 days + --> +<feed description="Raw customer email feed" name="oracle-feed-import-filesystem" xmlns="uri:falcon:feed:0.1"> + <tags>externalSystem=USWestEmailServers,classification=secure</tags> + <groups>DataImportPipeline</groups> + <frequency>minutes(10)</frequency> + <late-arrival cut-off="hours(4)"/> + <clusters> + <cluster name="primaryCluster" type="source"> + <validity start="2016-04-10T00:00Z" end="2017-03-31T00:00Z"/> + <retention limit="days(90)" action="delete"/> + <import> + <source name="oracle-db" tableName="DB_RAW_DATA"> + <extract type="full"> + <mergepolicy>snapshot</mergepolicy> + </extract> + <fields> + <includes> + <field>ID</field> + <field>NAME</field> + </includes> + </fields> + </source> + <arguments> + <argument name="--split-by" value="ID"/> + <argument name="--num-mappers" value="2"/> + </arguments> + </import> + </cluster> + </clusters> + + <locations> + <location type="data" path="/user/ambari-qa/falcon/demo/primary/oracle-feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/> + <location type="stats" path="/none"/> + <location type="meta" path="/none"/> + </locations> + + <ACL owner="ambari-qa" group="users" permission="0755"/> + <schema location="/none" provider="none"/> + + <lifecycle> + <retention-stage> + <frequency>days(1)</frequency> + <queue>default</queue> + <priority>LOW</priority> + <properties> + <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property> + </properties> + </retention-stage> + </lifecycle> +</feed> http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/pgsql-datasource.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/pgsql-datasource.xml b/examples/entity/importexport/pgsql-datasource.xml new file mode 100755 index 0000000..b161bd3 --- /dev/null +++ b/examples/entity/importexport/pgsql-datasource.xml @@ -0,0 +1,55 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<datasource colo="west-coast" description="Postgres database on west coast" type="postgres" name="pgsql-db" xmlns="uri:falcon:datasource:0.1"> + <tags>[email protected], [email protected]</tags> + <interfaces> + <!-- ***** read interface ***** --> + <interface type="readonly" endpoint="jdbc:postgresql://c6402:5432/test"> + <credential type="password-text"> + <userName>ambari-qa</userName> + <passwordText>sqoop</passwordText> + </credential> + </interface> + + <!-- ***** write interface ***** --> + <interface type="write" endpoint="jdbc:postgresql://c6402:5432/test"> + <credential type="password-text"> + <userName>ambari-qa</userName> + <passwordText>sqoop</passwordText> + </credential> + </interface> + + <!-- ***** default credential ***** --> + <credential type="password-alias"> + <userName>ambari-qa</userName> + <passwordAlias> + <alias>sqoop.password.alias</alias> + <providerPath>hdfs://c6401.ambari.apache.org:8020/user/ambari-qa/sqoop_password.jceks</providerPath> + </passwordAlias> + </credential> + </interfaces> + + <driver> + <clazz>org.postgresql.Driver</clazz> + <jar>/user/oozie/share/lib/lib_20160628052542/sqoop/postgresql-9.4.1208.jar</jar> + </driver> +</datasource> + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/pgsql-feed-import-filesystem.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/pgsql-feed-import-filesystem.xml b/examples/entity/importexport/pgsql-feed-import-filesystem.xml new file mode 100755 index 0000000..fa88cd1 --- /dev/null +++ b/examples/entity/importexport/pgsql-feed-import-filesystem.xml @@ -0,0 +1,70 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> +<!-- + A feed representing Hourly customer email data retained for 90 days + --> +<feed description="Raw customer email feed" name="pgsql-feed-import-filesystem" xmlns="uri:falcon:feed:0.1"> + <tags>externalSystem=USWestEmailServers,classification=secure</tags> + <groups>DataImportPipeline</groups> + <frequency>minutes(10)</frequency> + <late-arrival cut-off="hours(4)"/> + <clusters> + <cluster name="primaryCluster" type="source"> + <validity start="2016-04-10T00:00Z" end="2017-03-31T00:00Z"/> + <retention limit="days(90)" action="delete"/> + <import> + <source name="pgsql-db" tableName="db_raw_data"> + <extract type="full"> + <mergepolicy>snapshot</mergepolicy> + </extract> + <fields> + <includes> + <field>id</field> + <field>name</field> + </includes> + </fields> + </source> + <arguments> + <argument name="--split-by" value="id"/> + <argument name="--num-mappers" value="2"/> + </arguments> + </import> + </cluster> + </clusters> + + <locations> + <location type="data" path="/user/ambari-qa/falcon/demo/primary/pgsql-feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/> + <location type="stats" path="/none"/> + <location type="meta" path="/none"/> + </locations> + + <ACL owner="ambari-qa" group="users" permission="0755"/> + <schema location="/none" provider="none"/> + + <lifecycle> + <retention-stage> + <frequency>days(1)</frequency> + <queue>default</queue> + <priority>LOW</priority> + <properties> + <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property> + </properties> + </retention-stage> + </lifecycle> +</feed> http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/primary_cluster.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/primary_cluster.xml b/examples/entity/importexport/primary_cluster.xml new file mode 100755 index 0000000..4d43fd7 --- /dev/null +++ b/examples/entity/importexport/primary_cluster.xml @@ -0,0 +1,33 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> +<cluster colo="USWestOregon" description="oregonHadoopCluster" name="primaryCluster" xmlns="uri:falcon:cluster:0.1"> + <interfaces> + <interface type="readonly" endpoint="webhdfs://localhost:50070" version="2.2.0" /> + <interface type="write" endpoint="hdfs://localhost:8020" version="2.2.0" /> + <interface type="execute" endpoint="localhost:8050" version="2.2.0" /> + <interface type="workflow" endpoint="http://localhost:11000/oozie/"; version="4.0.0" /> + <interface type="messaging" endpoint="tcp://localhost:61616?daemon=true" version="5.1.6" /> + <interface type="registry" endpoint="thrift://localhost:9083" version="0.14.0" /> + </interfaces> + <locations> + <location name="staging" path="/apps/falcon/primaryCluster/staging" /> + <location name="temp" path="/tmp" /> + <location name="working" path="/apps/falcon/primaryCluster/working" /> + </locations> +</cluster> http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/td-datasource.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/td-datasource.xml b/examples/entity/importexport/td-datasource.xml new file mode 100755 index 0000000..a9eb4ed --- /dev/null +++ b/examples/entity/importexport/td-datasource.xml @@ -0,0 +1,56 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> +<datasource colo="west-coast" description="Teradata database on west coast" type="teradata" name="td-db" xmlns="uri:falcon:datasource:0.1"> + <tags>[email protected], [email protected]</tags> + <interfaces> + + <!-- ***** read interface ***** --> + <interface type="readonly" endpoint="jdbc:teradata://10.10.20.13/DATABASE=retail"> + <credential type="password-text"> + <userName>dbc</userName> + <passwordText>dbc</passwordText> + </credential> + </interface> + + <!-- ***** write interface ***** --> + <interface type="write" endpoint="jdbc:teradata://10.10.20.13/DATABASE=retail"> + <credential type="password-text"> + <userName>dbc</userName> + <passwordText>dbc</passwordText> + </credential> + </interface> + + <!-- ***** default credential ***** --> + <credential type="password-alias"> + <userName>dbc</userName> + <passwordAlias> + <alias>sqoop.password.alias</alias> + <providerPath>hdfs://c6401.ambari.apache.org:8020/user/ambari-qa/sqoop_password.jceks</providerPath> + </passwordAlias> + </credential> + + </interfaces> + + <driver> + <clazz>com.teradata.jdbc.TeraDriver</clazz> + <jar>/user/oozie/share/lib/lib_20160628052542/sqoop/terajdbc4.jar</jar> + <jar>/user/oozie/share/lib/lib_20160628052542/sqoop/tdgssconfig.jar</jar> + </driver> +</datasource> + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/td-feed-import-filesystem.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/td-feed-import-filesystem.xml b/examples/entity/importexport/td-feed-import-filesystem.xml new file mode 100755 index 0000000..6dd94ed --- /dev/null +++ b/examples/entity/importexport/td-feed-import-filesystem.xml @@ -0,0 +1,70 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> +<!-- + A feed representing Hourly customer email data retained for 90 days + --> +<feed description="Raw customer email feed" name="td-feed-import-filesystem" xmlns="uri:falcon:feed:0.1"> + <tags>externalSystem=USWestEmailServers,classification=secure</tags> + <groups>DataImportPipeline</groups> + <frequency>minutes(10)</frequency> + <late-arrival cut-off="hours(4)"/> + <clusters> + <cluster name="primaryCluster" type="source"> + <validity start="2016-04-10T00:00Z" end="2017-03-31T00:00Z"/> + <retention limit="days(90)" action="delete"/> + <import> + <source name="td-db" tableName="db_raw_data"> + <extract type="full"> + <mergepolicy>snapshot</mergepolicy> + </extract> + <fields> + <includes> + <field>id</field> + <field>name</field> + </includes> + </fields> + </source> + <arguments> + <argument name="--split-by" value="id"/> + <argument name="--num-mappers" value="2"/> + </arguments> + </import> + </cluster> + </clusters> + + <locations> + <location type="data" path="/user/ambari-qa/falcon/demo/primary/td-feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/> + <location type="stats" path="/none"/> + <location type="meta" path="/none"/> + </locations> + + <ACL owner="ambari-qa" group="users" permission="0755"/> + <schema location="/none" provider="none"/> + + <lifecycle> + <retention-stage> + <frequency>days(1)</frequency> + <queue>default</queue> + <priority>LOW</priority> + <properties> + <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property> + </properties> + </retention-stage> + </lifecycle> +</feed> http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java ---------------------------------------------------------------------- diff --git a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java index e7ca869..93f3d1f 100644 --- a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java +++ b/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java @@ -94,12 +94,15 @@ public class DatabaseExportWorkflowBuilder extends ExportWorkflowBuilder { StringBuilder sqoopArgs = new StringBuilder(); StringBuilder sqoopOptions = new StringBuilder(); - buildConnectArg(sqoopArgs, cluster).append(ImportExportCommon.ARG_SEPARATOR); - buildTableArg(sqoopArgs, cluster).append(ImportExportCommon.ARG_SEPARATOR); - Datasource datasource = DatasourceHelper.getDatasource(FeedHelper.getExportDatasourceName( - FeedHelper.getCluster(entity, cluster.getName()))); - ImportExportCommon.buildUserPasswordArg(sqoopArgs, sqoopOptions, datasource) - .append(ImportExportCommon.ARG_SEPARATOR); + org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName()); + Datasource datasource = DatasourceHelper.getDatasource(FeedHelper.getExportDatasourceName(feedCluster)); + ImportExportCommon.buildDriverArgs(sqoopArgs, datasource).append(ImportExportCommon.ARG_SEPARATOR); + ImportExportCommon.buildConnectArg(sqoopArgs, DatasourceHelper.getWriteEndpoint(datasource)) + .append(ImportExportCommon.ARG_SEPARATOR); + ImportExportCommon.buildTableArg(sqoopArgs, FeedHelper.getExportDataSourceTableName(feedCluster)) + .append(ImportExportCommon.ARG_SEPARATOR); + ImportExportCommon.buildUserPasswordArg(sqoopArgs, sqoopOptions, + DatasourceHelper.getWritePasswordInfo(datasource)).append(ImportExportCommon.ARG_SEPARATOR); buildNumMappers(sqoopArgs, extraArgs).append(ImportExportCommon.ARG_SEPARATOR); buildArguments(sqoopArgs, extraArgs, feed, cluster).append(ImportExportCommon.ARG_SEPARATOR); buildLoadType(sqoopArgs, cluster).append(ImportExportCommon.ARG_SEPARATOR); @@ -111,19 +114,6 @@ public class DatabaseExportWorkflowBuilder extends ExportWorkflowBuilder { .append(sqoopArgs).toString(); } - private StringBuilder buildConnectArg(StringBuilder builder, Cluster cluster) throws FalconException { - org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName()); - return builder.append("--connect").append(ImportExportCommon.ARG_SEPARATOR) - .append(DatasourceHelper.getReadOnlyEndpoint( - DatasourceHelper.getDatasource(FeedHelper.getExportDatasourceName(feedCluster)))); - } - - private StringBuilder buildTableArg(StringBuilder builder, Cluster cluster) throws FalconException { - org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName()); - return builder.append("--table").append(ImportExportCommon.ARG_SEPARATOR) - .append(FeedHelper.getExportDataSourceTableName(feedCluster)); - } - private StringBuilder buildLoadType(StringBuilder builder, Cluster cluster) throws FalconException { org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName()); http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java ---------------------------------------------------------------------- diff --git a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java index 1efa368..77d5462 100644 --- a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java +++ b/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java @@ -95,13 +95,16 @@ public class DatabaseImportWorkflowBuilder extends ImportWorkflowBuilder { Map<String, String> extraArgs = getArguments(cluster); StringBuilder sqoopArgs = new StringBuilder(); StringBuilder sqoopOptions = new StringBuilder(); - buildDriverArgs(sqoopArgs, cluster).append(ImportExportCommon.ARG_SEPARATOR); - buildConnectArg(sqoopArgs, cluster).append(ImportExportCommon.ARG_SEPARATOR); - buildTableArg(sqoopArgs, cluster).append(ImportExportCommon.ARG_SEPARATOR); - Datasource datasource = DatasourceHelper.getDatasource(FeedHelper.getImportDatasourceName( - FeedHelper.getCluster(entity, cluster.getName()))); - ImportExportCommon.buildUserPasswordArg(sqoopArgs, sqoopOptions, datasource) + + org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName()); + Datasource datasource = DatasourceHelper.getDatasource(FeedHelper.getImportDatasourceName(feedCluster)); + ImportExportCommon.buildDriverArgs(sqoopArgs, datasource).append(ImportExportCommon.ARG_SEPARATOR); + ImportExportCommon.buildConnectArg(sqoopArgs, DatasourceHelper.getReadOnlyEndpoint(datasource)) .append(ImportExportCommon.ARG_SEPARATOR); + ImportExportCommon.buildTableArg(sqoopArgs, FeedHelper.getImportDataSourceTableName(feedCluster)) + .append(ImportExportCommon.ARG_SEPARATOR); + ImportExportCommon.buildUserPasswordArg(sqoopArgs, sqoopOptions, + DatasourceHelper.getReadPasswordInfo(datasource)).append(ImportExportCommon.ARG_SEPARATOR); buildNumMappers(sqoopArgs, extraArgs).append(ImportExportCommon.ARG_SEPARATOR); buildArguments(sqoopArgs, extraArgs).append(ImportExportCommon.ARG_SEPARATOR); buildTargetArg(sqoopArgs, feed, cluster).append(ImportExportCommon.ARG_SEPARATOR); @@ -112,28 +115,6 @@ public class DatabaseImportWorkflowBuilder extends ImportWorkflowBuilder { .append(sqoopArgs).toString(); } - private StringBuilder buildDriverArgs(StringBuilder builder, Cluster cluster) throws FalconException { - org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName()); - Datasource db = DatasourceHelper.getDatasource(FeedHelper.getImportDatasourceName(feedCluster)); - if ((db.getDriver() != null) && (db.getDriver().getClazz() != null)) { - builder.append("--driver").append(ImportExportCommon.ARG_SEPARATOR).append(db.getDriver().getClazz()); - } - return builder; - } - - private StringBuilder buildConnectArg(StringBuilder builder, Cluster cluster) throws FalconException { - org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName()); - return builder.append("--connect").append(ImportExportCommon.ARG_SEPARATOR) - .append(DatasourceHelper.getReadOnlyEndpoint( - DatasourceHelper.getDatasource(FeedHelper.getImportDatasourceName(feedCluster)))); - } - - private StringBuilder buildTableArg(StringBuilder builder, Cluster cluster) throws FalconException { - org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName()); - return builder.append("--table").append(ImportExportCommon.ARG_SEPARATOR) - .append(FeedHelper.getImportDataSourceTableName(feedCluster)); - } - private StringBuilder buildTargetArg(StringBuilder builder, Feed feed, Cluster cluster) throws FalconException { Storage.TYPE feedStorageType = FeedHelper.getStorageType(feed, cluster);
