http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties ---------------------------------------------------------------------- diff --git a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties deleted file mode 100644 index 4642835..0000000 --- a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties +++ /dev/null @@ -1,79 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -##### NOTE: This is a TEMPLATE file which can be copied and edited - -##### Recipe properties -##### Unique recipe job name -falcon.recipe.name=sales-monthly - -##### Workflow properties -falcon.recipe.workflow.name=hdfs-dr-workflow -# Provide Wf absolute path. This can be HDFS or local FS path. If WF is on local FS it will be copied to HDFS -falcon.recipe.workflow.path=/apps/data-mirroring/workflows/hdfs-replication-workflow.xml -# Provide Wf lib absolute path. This can be HDFS or local FS path. If libs are on local FS it will be copied to HDFS -#falcon.recipe.workflow.lib.path=/recipes/hdfs-replication/lib - -##### Cluster properties -# Cluster where job should run -falcon.recipe.cluster.name=primaryCluster -# Change the cluster hdfs write end point here. This is mandatory. -falcon.recipe.cluster.hdfs.writeEndPoint=hdfs://240.0.0.10:8020 -# Change the cluster validity start time here -falcon.recipe.cluster.validity.start=2015-03-13T00:00Z -# Change the cluster validity end time here -falcon.recipe.cluster.validity.end=2016-12-30T00:00Z - -##### Scheduling properties -# Change the recipe frequency here. Valid frequency type are minutes, hours, days, months -falcon.recipe.process.frequency=minutes(5) - -##### Tag properties - An optional list of comma separated tags, Key Value Pairs, separated by comma -##### Uncomment to add tags -#falcon.recipe.tags= - -##### Retry policy properties - -falcon.recipe.retry.policy=periodic -falcon.recipe.retry.delay=minutes(30) -falcon.recipe.retry.attempts=3 -falcon.recipe.retry.onTimeout=false - -##### ACL properties - Uncomment and change ACL if authorization is enabled - -falcon.recipe.acl.owner=ambari-qa -falcon.recipe.acl.group=users -falcon.recipe.acl.permission=0x755 -falcon.recipe.nn.principal=nn/[email protected] - -##### Custom Job properties - -# Specify multiple comma separated source directories -drSourceDir=/user/hrt_qa/dr/test/primaryCluster/input -drSourceClusterFS=hdfs://240.0.0.10:8020 -drTargetDir=/user/hrt_qa/dr/test/backupCluster/input -drTargetClusterFS=hdfs://240.0.0.11:8020 - -# Change it to specify the maximum number of mappers for DistCP -distcpMaxMaps=1 -# Change it to specify the bandwidth in MB for each mapper in DistCP -distcpMapBandwidth=100 - -##### Email Notification for Falcon instance completion -falcon.recipe.notification.type=email -falcon.recipe.notification.receivers=NA \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/addons/recipes/hive-disaster-recovery/README.txt ---------------------------------------------------------------------- diff --git a/addons/recipes/hive-disaster-recovery/README.txt b/addons/recipes/hive-disaster-recovery/README.txt deleted file mode 100644 index ab393b1..0000000 --- a/addons/recipes/hive-disaster-recovery/README.txt +++ /dev/null @@ -1,58 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -Hive Metastore Disaster Recovery Recipe - -Overview -This recipe implements replicating hive metadata and data from one -Hadoop cluster to another Hadoop cluster. -This piggy backs on replication solution in Falcon which uses the DistCp tool. - -Use Case -* -* - -Limitations -* -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -Hive Metastore Disaster Recovery Recipe - -Overview -This recipe implements replicating hive metadata and data from one -Hadoop cluster to another Hadoop cluster. -This piggy backs on replication solution in Falcon which uses the DistCp tool. - -Use Case -* -* - -Limitations -* http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/addons/recipes/hive-disaster-recovery/pom.xml ---------------------------------------------------------------------- diff --git a/addons/recipes/hive-disaster-recovery/pom.xml b/addons/recipes/hive-disaster-recovery/pom.xml deleted file mode 100644 index 0f782d2..0000000 --- a/addons/recipes/hive-disaster-recovery/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> - -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> - -<project xmlns="http://maven.apache.org/POM/4.0.0"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> - - <modelVersion>4.0.0</modelVersion> - <groupId>org.apache.falcon.recipes</groupId> - <artifactId>falcon-hive-replication-recipe</artifactId> - <version>0.10-SNAPSHOT</version> - <description>Apache Falcon Hive Disaster Recovery Recipe</description> - <name>Apache Falcon Sample Hive Disaster Recovery Recipe</name> - <packaging>jar</packaging> -</project> http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-template.xml ---------------------------------------------------------------------- diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-template.xml b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-template.xml deleted file mode 100644 index f0de091..0000000 --- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-template.xml +++ /dev/null @@ -1,45 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> - -<process name="##name##" xmlns="uri:falcon:process:0.1"> - <clusters> - <!-- source --> - <cluster name="##cluster.name##"> - <validity end="##cluster.validity.end##" start="##cluster.validity.start##"/> - </cluster> - </clusters> - - <tags>_falcon_mirroring_type=HIVE</tags> - - <parallel>1</parallel> - <!-- Replication needs to run only once to catch up --> - <order>LAST_ONLY</order> - <frequency>##process.frequency##</frequency> - <timezone>UTC</timezone> - - <properties> - <property name="oozie.wf.subworkflow.classpath.inheritance" value="true"/> - </properties> - - <workflow name="##workflow.name##" engine="oozie" - path="/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml" lib="##workflow.lib.path##"/> - <retry policy="##retry.policy##" delay="##retry.delay##" attempts="3"/> - <notification type="##notification.type##" to="##notification.receivers##"/> - <ACL/> -</process> http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-workflow.xml ---------------------------------------------------------------------- diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-workflow.xml b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-workflow.xml deleted file mode 100644 index 2d6b8be..0000000 --- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-workflow.xml +++ /dev/null @@ -1,363 +0,0 @@ -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> -<workflow-app xmlns='uri:oozie:workflow:0.3' name='falcon-dr-hive-workflow'> - <credentials> - <credential name='hive_src_credentials' type='hcat'> - <property> - <name>hcat.metastore.uri</name> - <value>${sourceMetastoreUri}</value> - </property> - <property> - <name>hcat.metastore.principal</name> - <value>${sourceHiveMetastoreKerberosPrincipal}</value> - </property> - </credential> - <credential name='hive_tgt_credentials' type='hcat'> - <property> - <name>hcat.metastore.uri</name> - <value>${targetMetastoreUri}</value> - </property> - <property> - <name>hcat.metastore.principal</name> - <value>${targetHiveMetastoreKerberosPrincipal}</value> - </property> - </credential> - <credential name="hive2_src_credentials" type="hive2"> - <property> - <name>hive2.server.principal</name> - <value>${sourceHive2KerberosPrincipal}</value> - </property> - <property> - <name>hive2.jdbc.url</name> - <value>jdbc:${sourceHiveServer2Uri}/${sourceDatabase}</value> - </property> - </credential> - <credential name="hive2_tgt_credentials" type="hive2"> - <property> - <name>hive2.server.principal</name> - <value>${targetHive2KerberosPrincipal}</value> - </property> - <property> - <name>hive2.jdbc.url</name> - <value>jdbc:${targetHiveServer2Uri}/${sourceDatabase}</value> - </property> - </credential> - </credentials> - <start to='last-event'/> - <action name="last-event" cred="hive_tgt_credentials"> - <java> - <job-tracker>${jobTracker}</job-tracker> - <name-node>${nameNode}</name-node> - <configuration> - <property> <!-- hadoop 2 parameter --> - <name>oozie.launcher.mapreduce.job.user.classpath.first</name> - <value>true</value> - </property> - <property> - <name>mapred.job.queue.name</name> - <value>${queueName}</value> - </property> - <property> - <name>oozie.launcher.mapred.job.priority</name> - <value>${jobPriority}</value> - </property> - <property> - <name>oozie.use.system.libpath</name> - <value>true</value> - </property> - <property> - <name>oozie.action.sharelib.for.java</name> - <value>distcp,hive,hive2,hcatalog</value> - </property> - <property> - <name>oozie.launcher.mapreduce.job.hdfs-servers</name> - <value>${sourceNN},${targetNN}</value> - </property> - <property> - <name>mapreduce.job.hdfs-servers</name> - <value>${sourceNN},${targetNN}</value> - </property> - </configuration> - <main-class>org.apache.falcon.hive.HiveDRTool</main-class> - <arg>-Dmapred.job.queue.name=${queueName}</arg> - <arg>-Dmapred.job.priority=${jobPriority}</arg> - <arg>-falconLibPath</arg> - <arg>${wf:conf("falcon.libpath")}</arg> - <arg>-sourceCluster</arg> - <arg>${sourceCluster}</arg> - <arg>-sourceMetastoreUri</arg> - <arg>${sourceMetastoreUri}</arg> - <arg>-sourceHiveServer2Uri</arg> - <arg>${sourceHiveServer2Uri}</arg> - <arg>-sourceDatabase</arg> - <arg>${sourceDatabase}</arg> - <arg>-sourceTable</arg> - <arg>${sourceTable}</arg> - <arg>-sourceStagingPath</arg> - <arg>${sourceStagingPath}</arg> - <arg>-sourceNN</arg> - <arg>${sourceNN}</arg> - <arg>-sourceNNKerberosPrincipal</arg> - <arg>${sourceNNKerberosPrincipal}</arg> - <arg>-sourceHiveMetastoreKerberosPrincipal</arg> - <arg>${sourceHiveMetastoreKerberosPrincipal}</arg> - <arg>-sourceHive2KerberosPrincipal</arg> - <arg>${sourceHive2KerberosPrincipal}</arg> - <arg>-targetCluster</arg> - <arg>${targetCluster}</arg> - <arg>-targetMetastoreUri</arg> - <arg>${targetMetastoreUri}</arg> - <arg>-targetHiveServer2Uri</arg> - <arg>${targetHiveServer2Uri}</arg> - <arg>-targetStagingPath</arg> - <arg>${targetStagingPath}</arg> - <arg>-targetNN</arg> - <arg>${targetNN}</arg> - <arg>-targetNNKerberosPrincipal</arg> - <arg>${targetNNKerberosPrincipal}</arg> - <arg>-targetHiveMetastoreKerberosPrincipal</arg> - <arg>${targetHiveMetastoreKerberosPrincipal}</arg> - <arg>-targetHive2KerberosPrincipal</arg> - <arg>${targetHive2KerberosPrincipal}</arg> - <arg>-maxEvents</arg> - <arg>${maxEvents}</arg> - <arg>-clusterForJobRun</arg> - <arg>${clusterForJobRun}</arg> - <arg>-clusterForJobRunWriteEP</arg> - <arg>${clusterForJobRunWriteEP}</arg> - <arg>-clusterForJobNNKerberosPrincipal</arg> - <arg>${clusterForJobNNKerberosPrincipal}</arg> - <arg>-tdeEncryptionEnabled</arg> - <arg>${tdeEncryptionEnabled}</arg> - <arg>-drJobName</arg> - <arg>${drJobName}-${nominalTime}</arg> - <arg>-executionStage</arg> - <arg>lastevents</arg> - </java> - <ok to="export-dr-replication"/> - <error to="fail"/> - </action> - <!-- Export Replication action --> - <action name="export-dr-replication" cred="hive_src_credentials,hive2_src_credentials"> - <java> - <job-tracker>${jobTracker}</job-tracker> - <name-node>${nameNode}</name-node> - <configuration> - <property> <!-- hadoop 2 parameter --> - <name>oozie.launcher.mapreduce.job.user.classpath.first</name> - <value>true</value> - </property> - <property> - <name>mapred.job.queue.name</name> - <value>${queueName}</value> - </property> - <property> - <name>oozie.launcher.mapred.job.priority</name> - <value>${jobPriority}</value> - </property> - <property> - <name>oozie.use.system.libpath</name> - <value>true</value> - </property> - <property> - <name>oozie.action.sharelib.for.java</name> - <value>distcp,hive,hive2,hcatalog</value> - </property> - <property> - <name>oozie.launcher.mapreduce.job.hdfs-servers</name> - <value>${sourceNN},${targetNN}</value> - </property> - <property> - <name>mapreduce.job.hdfs-servers</name> - <value>${sourceNN},${targetNN}</value> - </property> - </configuration> - <main-class>org.apache.falcon.hive.HiveDRTool</main-class> - <arg>-Dmapred.job.queue.name=${queueName}</arg> - <arg>-Dmapred.job.priority=${jobPriority}</arg> - <arg>-falconLibPath</arg> - <arg>${wf:conf("falcon.libpath")}</arg> - <arg>-replicationMaxMaps</arg> - <arg>${replicationMaxMaps}</arg> - <arg>-distcpMaxMaps</arg> - <arg>${distcpMaxMaps}</arg> - <arg>-sourceCluster</arg> - <arg>${sourceCluster}</arg> - <arg>-sourceMetastoreUri</arg> - <arg>${sourceMetastoreUri}</arg> - <arg>-sourceHiveServer2Uri</arg> - <arg>${sourceHiveServer2Uri}</arg> - <arg>-sourceDatabase</arg> - <arg>${sourceDatabase}</arg> - <arg>-sourceTable</arg> - <arg>${sourceTable}</arg> - <arg>-sourceStagingPath</arg> - <arg>${sourceStagingPath}</arg> - <arg>-sourceNN</arg> - <arg>${sourceNN}</arg> - <arg>-sourceNNKerberosPrincipal</arg> - <arg>${sourceNNKerberosPrincipal}</arg> - <arg>-sourceHiveMetastoreKerberosPrincipal</arg> - <arg>${sourceHiveMetastoreKerberosPrincipal}</arg> - <arg>-sourceHive2KerberosPrincipal</arg> - <arg>${sourceHive2KerberosPrincipal}</arg> - <arg>-targetCluster</arg> - <arg>${targetCluster}</arg> - <arg>-targetMetastoreUri</arg> - <arg>${targetMetastoreUri}</arg> - <arg>-targetHiveServer2Uri</arg> - <arg>${targetHiveServer2Uri}</arg> - <arg>-targetStagingPath</arg> - <arg>${targetStagingPath}</arg> - <arg>-targetNN</arg> - <arg>${targetNN}</arg> - <arg>-targetNNKerberosPrincipal</arg> - <arg>${targetNNKerberosPrincipal}</arg> - <arg>-targetHiveMetastoreKerberosPrincipal</arg> - <arg>${targetHiveMetastoreKerberosPrincipal}</arg> - <arg>-targetHive2KerberosPrincipal</arg> - <arg>${targetHive2KerberosPrincipal}</arg> - <arg>-maxEvents</arg> - <arg>${maxEvents}</arg> - <arg>-distcpMapBandwidth</arg> - <arg>${distcpMapBandwidth}</arg> - <arg>-clusterForJobRun</arg> - <arg>${clusterForJobRun}</arg> - <arg>-clusterForJobRunWriteEP</arg> - <arg>${clusterForJobRunWriteEP}</arg> - <arg>-clusterForJobNNKerberosPrincipal</arg> - <arg>${clusterForJobNNKerberosPrincipal}</arg> - <arg>-tdeEncryptionEnabled</arg> - <arg>${tdeEncryptionEnabled}</arg> - <arg>-drJobName</arg> - <arg>${drJobName}-${nominalTime}</arg> - <arg>-executionStage</arg> - <arg>export</arg> - <arg>-counterLogDir</arg> - <arg>${logDir}/job-${nominalTime}/${srcClusterName == 'NA' ? '' : srcClusterName}/</arg> - </java> - <ok to="import-dr-replication"/> - <error to="fail"/> - </action> - <!-- Import Replication action --> - <action name="import-dr-replication" cred="hive_tgt_credentials,hive2_tgt_credentials"> - <java> - <job-tracker>${jobTracker}</job-tracker> - <name-node>${nameNode}</name-node> - <configuration> - <property> <!-- hadoop 2 parameter --> - <name>oozie.launcher.mapreduce.job.user.classpath.first</name> - <value>true</value> - </property> - <property> - <name>mapred.job.queue.name</name> - <value>${queueName}</value> - </property> - <property> - <name>oozie.launcher.mapred.job.priority</name> - <value>${jobPriority}</value> - </property> - <property> - <name>oozie.use.system.libpath</name> - <value>true</value> - </property> - <property> - <name>oozie.action.sharelib.for.java</name> - <value>distcp,hive,hive2,hcatalog</value> - </property> - <property> - <name>oozie.launcher.mapreduce.job.hdfs-servers</name> - <value>${sourceNN},${targetNN}</value> - </property> - <property> - <name>mapreduce.job.hdfs-servers</name> - <value>${sourceNN},${targetNN}</value> - </property> - </configuration> - <main-class>org.apache.falcon.hive.HiveDRTool</main-class> - <arg>-Dmapred.job.queue.name=${queueName}</arg> - <arg>-Dmapred.job.priority=${jobPriority}</arg> - <arg>-falconLibPath</arg> - <arg>${wf:conf("falcon.libpath")}</arg> - <arg>-replicationMaxMaps</arg> - <arg>${replicationMaxMaps}</arg> - <arg>-distcpMaxMaps</arg> - <arg>${distcpMaxMaps}</arg> - <arg>-sourceCluster</arg> - <arg>${sourceCluster}</arg> - <arg>-sourceMetastoreUri</arg> - <arg>${sourceMetastoreUri}</arg> - <arg>-sourceHiveServer2Uri</arg> - <arg>${sourceHiveServer2Uri}</arg> - <arg>-sourceDatabase</arg> - <arg>${sourceDatabase}</arg> - <arg>-sourceTable</arg> - <arg>${sourceTable}</arg> - <arg>-sourceStagingPath</arg> - <arg>${sourceStagingPath}</arg> - <arg>-sourceNN</arg> - <arg>${sourceNN}</arg> - <arg>-sourceNNKerberosPrincipal</arg> - <arg>${sourceNNKerberosPrincipal}</arg> - <arg>-sourceHiveMetastoreKerberosPrincipal</arg> - <arg>${sourceHiveMetastoreKerberosPrincipal}</arg> - <arg>-sourceHive2KerberosPrincipal</arg> - <arg>${sourceHive2KerberosPrincipal}</arg> - <arg>-targetCluster</arg> - <arg>${targetCluster}</arg> - <arg>-targetMetastoreUri</arg> - <arg>${targetMetastoreUri}</arg> - <arg>-targetHiveServer2Uri</arg> - <arg>${targetHiveServer2Uri}</arg> - <arg>-targetStagingPath</arg> - <arg>${targetStagingPath}</arg> - <arg>-targetNN</arg> - <arg>${targetNN}</arg> - <arg>-targetNNKerberosPrincipal</arg> - <arg>${targetNNKerberosPrincipal}</arg> - <arg>-targetHiveMetastoreKerberosPrincipal</arg> - <arg>${targetHiveMetastoreKerberosPrincipal}</arg> - <arg>-targetHive2KerberosPrincipal</arg> - <arg>${targetHive2KerberosPrincipal}</arg> - <arg>-maxEvents</arg> - <arg>${maxEvents}</arg> - <arg>-distcpMapBandwidth</arg> - <arg>${distcpMapBandwidth}</arg> - <arg>-clusterForJobRun</arg> - <arg>${clusterForJobRun}</arg> - <arg>-clusterForJobRunWriteEP</arg> - <arg>${clusterForJobRunWriteEP}</arg> - <arg>-clusterForJobNNKerberosPrincipal</arg> - <arg>${clusterForJobNNKerberosPrincipal}</arg> - <arg>-tdeEncryptionEnabled</arg> - <arg>${tdeEncryptionEnabled}</arg> - <arg>-drJobName</arg> - <arg>${drJobName}-${nominalTime}</arg> - <arg>-executionStage</arg> - <arg>import</arg> - </java> - <ok to="end"/> - <error to="fail"/> - </action> - <kill name="fail"> - <message> - Workflow action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] - </message> - </kill> - <end name="end"/> -</workflow-app> http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties ---------------------------------------------------------------------- diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties deleted file mode 100644 index 331d57e..0000000 --- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties +++ /dev/null @@ -1,110 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -##### NOTE: This is a TEMPLATE file which can be copied and edited - -##### Recipe properties -falcon.recipe.name=hive-disaster-recovery - - -##### Workflow properties -falcon.recipe.workflow.name=hive-dr-workflow -# Provide Wf absolute path. This can be HDFS or local FS path. If WF is on local FS it will be copied to HDFS -falcon.recipe.workflow.path=/recipes/hive-replication/hive-disaster-recovery-secure-workflow.xml - -##### Cluster properties - -# Change the cluster name where replication job should run here -falcon.recipe.cluster.name=backupCluster -# Change the cluster hdfs write end point here. This is mandatory. -falcon.recipe.cluster.hdfs.writeEndPoint=hdfs://localhost:8020 -# Change the cluster validity start time here -falcon.recipe.cluster.validity.start=2014-10-01T00:00Z -# Change the cluster validity end time here -falcon.recipe.cluster.validity.end=2016-12-30T00:00Z -# Change the cluster namenode kerberos principal. This is mandatory on secure clusters. -falcon.recipe.nn.principal=nn/[email protected] - -##### Scheduling properties - -# Change the process frequency here. Valid frequency type are minutes, hours, days, months -falcon.recipe.process.frequency=minutes(60) - -##### Retry policy properties - -falcon.recipe.retry.policy=periodic -falcon.recipe.retry.delay=minutes(30) -falcon.recipe.retry.attempts=3 -falcon.recipe.retry.onTimeout=false - -##### Tag properties - An optional list of comma separated tags, Key Value Pairs, separated by comma -##### Uncomment to add tags -#falcon.recipe.tags=owner=landing,pipeline=adtech - -##### ACL properties - Uncomment and change ACL if authorization is enabled - -#falcon.recipe.acl.owner=testuser -#falcon.recipe.acl.group=group -#falcon.recipe.acl.permission=0x755 - -##### Custom Job properties - -##### Source Cluster DR properties -sourceCluster=primaryCluster -sourceMetastoreUri=thrift://localhost:9083 -sourceHiveServer2Uri=hive2://localhost:10000 -# For DB level replicaiton to replicate multiple databases specify comma separated list of tables -sourceDatabase=default -# For DB level replication specify * for sourceTable. -# For table level replication to replicate multiple tables specify comma separated list of tables -sourceTable=testtable_dr -## Please specify staging dir in the source without fully qualified domain name. -sourceStagingPath=/apps/hive/tools/dr -sourceNN=hdfs://localhost:8020 -# Specify kerberos principal required to access source namenode and hive servers, optional on non-secure cluster. -sourceNNKerberosPrincipal=nn/[email protected] -sourceHiveMetastoreKerberosPrincipal=hive/[email protected] -sourceHive2KerberosPrincipal=hive/[email protected] - -##### Target Cluster DR properties -targetCluster=backupCluster -targetMetastoreUri=thrift://localhost:9083 -targetHiveServer2Uri=hive2://localhost:10000 -## Please specify staging dir in the target without fully qualified domain name. -targetStagingPath=/apps/hive/tools/dr -targetNN=hdfs://localhost:8020 -# Specify kerberos principal required to access target namenode and hive servers, optional on non-secure cluster. -targetNNKerberosPrincipal=nn/[email protected] -targetHiveMetastoreKerberosPrincipal=hive/[email protected] -targetHive2KerberosPrincipal=hive/[email protected] - -# To ceil the max events processed each time job runs. Set it to max value depending on your bandwidth limit. -# Setting it to -1 will process all the events but can hog up the bandwidth. Use it judiciously! -maxEvents=-1 -# Change it to specify the maximum number of mappers for replication -replicationMaxMaps=5 -# Change it to specify the maximum number of mappers for DistCP -distcpMaxMaps=1 -# Change it to specify the bandwidth in MB for each mapper in DistCP -distcpMapBandwidth=100 -# Set this flag to true if TDE encryption is enabled on source and target -tdeEncryptionEnabled=false - -##### Email Notification for Falcon instance completion -falcon.recipe.notification.type=email -falcon.recipe.notification.receivers=NA \ No newline at end of file http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-template.xml ---------------------------------------------------------------------- diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-template.xml b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-template.xml deleted file mode 100644 index f0de091..0000000 --- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-template.xml +++ /dev/null @@ -1,45 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> - -<process name="##name##" xmlns="uri:falcon:process:0.1"> - <clusters> - <!-- source --> - <cluster name="##cluster.name##"> - <validity end="##cluster.validity.end##" start="##cluster.validity.start##"/> - </cluster> - </clusters> - - <tags>_falcon_mirroring_type=HIVE</tags> - - <parallel>1</parallel> - <!-- Replication needs to run only once to catch up --> - <order>LAST_ONLY</order> - <frequency>##process.frequency##</frequency> - <timezone>UTC</timezone> - - <properties> - <property name="oozie.wf.subworkflow.classpath.inheritance" value="true"/> - </properties> - - <workflow name="##workflow.name##" engine="oozie" - path="/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml" lib="##workflow.lib.path##"/> - <retry policy="##retry.policy##" delay="##retry.delay##" attempts="3"/> - <notification type="##notification.type##" to="##notification.receivers##"/> - <ACL/> -</process> http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml ---------------------------------------------------------------------- diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml deleted file mode 100644 index 296e049..0000000 --- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml +++ /dev/null @@ -1,249 +0,0 @@ -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> -<workflow-app xmlns='uri:oozie:workflow:0.3' name='falcon-dr-hive-workflow'> - <start to='last-event'/> - <action name="last-event"> - <java> - <job-tracker>${jobTracker}</job-tracker> - <name-node>${nameNode}</name-node> - <configuration> - <property> <!-- hadoop 2 parameter --> - <name>oozie.launcher.mapreduce.job.user.classpath.first</name> - <value>true</value> - </property> - <property> - <name>mapred.job.queue.name</name> - <value>${queueName}</value> - </property> - <property> - <name>oozie.launcher.mapred.job.priority</name> - <value>${jobPriority}</value> - </property> - <property> - <name>oozie.use.system.libpath</name> - <value>true</value> - </property> - <property> - <name>oozie.action.sharelib.for.java</name> - <value>distcp,hive,hive2,hcatalog</value> - </property> - </configuration> - <main-class>org.apache.falcon.hive.HiveDRTool</main-class> - <arg>-Dmapred.job.queue.name=${queueName}</arg> - <arg>-Dmapred.job.priority=${jobPriority}</arg> - <arg>-falconLibPath</arg> - <arg>${wf:conf("falcon.libpath")}</arg> - <arg>-sourceCluster</arg> - <arg>${sourceCluster}</arg> - <arg>-sourceMetastoreUri</arg> - <arg>${sourceMetastoreUri}</arg> - <arg>-sourceHiveServer2Uri</arg> - <arg>${sourceHiveServer2Uri}</arg> - <arg>-sourceDatabase</arg> - <arg>${sourceDatabase}</arg> - <arg>-sourceTable</arg> - <arg>${sourceTable}</arg> - <arg>-sourceStagingPath</arg> - <arg>${sourceStagingPath}</arg> - <arg>-sourceNN</arg> - <arg>${sourceNN}</arg> - <arg>-targetCluster</arg> - <arg>${targetCluster}</arg> - <arg>-targetMetastoreUri</arg> - <arg>${targetMetastoreUri}</arg> - <arg>-targetHiveServer2Uri</arg> - <arg>${targetHiveServer2Uri}</arg> - <arg>-targetStagingPath</arg> - <arg>${targetStagingPath}</arg> - <arg>-targetNN</arg> - <arg>${targetNN}</arg> - <arg>-maxEvents</arg> - <arg>${maxEvents}</arg> - <arg>-clusterForJobRun</arg> - <arg>${clusterForJobRun}</arg> - <arg>-clusterForJobRunWriteEP</arg> - <arg>${clusterForJobRunWriteEP}</arg> - <arg>-drJobName</arg> - <arg>${drJobName}-${nominalTime}</arg> - <arg>-executionStage</arg> - <arg>lastevents</arg> - </java> - <ok to="export-dr-replication"/> - <error to="fail"/> - </action> - <!-- Export Replication action --> - <action name="export-dr-replication"> - <java> - <job-tracker>${jobTracker}</job-tracker> - <name-node>${nameNode}</name-node> - <configuration> - <property> <!-- hadoop 2 parameter --> - <name>oozie.launcher.mapreduce.job.user.classpath.first</name> - <value>true</value> - </property> - <property> - <name>mapred.job.queue.name</name> - <value>${queueName}</value> - </property> - <property> - <name>oozie.launcher.mapred.job.priority</name> - <value>${jobPriority}</value> - </property> - <property> - <name>oozie.use.system.libpath</name> - <value>true</value> - </property> - <property> - <name>oozie.action.sharelib.for.java</name> - <value>distcp,hive,hive2,hcatalog</value> - </property> - </configuration> - <main-class>org.apache.falcon.hive.HiveDRTool</main-class> - <arg>-Dmapred.job.queue.name=${queueName}</arg> - <arg>-Dmapred.job.priority=${jobPriority}</arg> - <arg>-falconLibPath</arg> - <arg>${wf:conf("falcon.libpath")}</arg> - <arg>-replicationMaxMaps</arg> - <arg>${replicationMaxMaps}</arg> - <arg>-distcpMaxMaps</arg> - <arg>${distcpMaxMaps}</arg> - <arg>-sourceCluster</arg> - <arg>${sourceCluster}</arg> - <arg>-sourceMetastoreUri</arg> - <arg>${sourceMetastoreUri}</arg> - <arg>-sourceHiveServer2Uri</arg> - <arg>${sourceHiveServer2Uri}</arg> - <arg>-sourceDatabase</arg> - <arg>${sourceDatabase}</arg> - <arg>-sourceTable</arg> - <arg>${sourceTable}</arg> - <arg>-sourceStagingPath</arg> - <arg>${sourceStagingPath}</arg> - <arg>-sourceNN</arg> - <arg>${sourceNN}</arg> - <arg>-targetCluster</arg> - <arg>${targetCluster}</arg> - <arg>-targetMetastoreUri</arg> - <arg>${targetMetastoreUri}</arg> - <arg>-targetHiveServer2Uri</arg> - <arg>${targetHiveServer2Uri}</arg> - <arg>-targetStagingPath</arg> - <arg>${targetStagingPath}</arg> - <arg>-targetNN</arg> - <arg>${targetNN}</arg> - <arg>-maxEvents</arg> - <arg>${maxEvents}</arg> - <arg>-distcpMapBandwidth</arg> - <arg>${distcpMapBandwidth}</arg> - <arg>-clusterForJobRun</arg> - <arg>${clusterForJobRun}</arg> - <arg>-clusterForJobRunWriteEP</arg> - <arg>${clusterForJobRunWriteEP}</arg> - <arg>-drJobName</arg> - <arg>${drJobName}-${nominalTime}</arg> - <arg>-executionStage</arg> - <arg>export</arg> - <arg>-counterLogDir</arg> - <arg>${logDir}/job-${nominalTime}/${srcClusterName == 'NA' ? '' : srcClusterName}/</arg> - </java> - <ok to="import-dr-replication"/> - <error to="fail"/> - </action> - <!-- Import Replication action --> - <action name="import-dr-replication"> - <java> - <job-tracker>${jobTracker}</job-tracker> - <name-node>${nameNode}</name-node> - <configuration> - <property> <!-- hadoop 2 parameter --> - <name>oozie.launcher.mapreduce.job.user.classpath.first</name> - <value>true</value> - </property> - <property> - <name>mapred.job.queue.name</name> - <value>${queueName}</value> - </property> - <property> - <name>oozie.launcher.mapred.job.priority</name> - <value>${jobPriority}</value> - </property> - <property> - <name>oozie.use.system.libpath</name> - <value>true</value> - </property> - <property> - <name>oozie.action.sharelib.for.java</name> - <value>distcp,hive,hive2,hcatalog</value> - </property> - </configuration> - <main-class>org.apache.falcon.hive.HiveDRTool</main-class> - <arg>-Dmapred.job.queue.name=${queueName}</arg> - <arg>-Dmapred.job.priority=${jobPriority}</arg> - <arg>-falconLibPath</arg> - <arg>${wf:conf("falcon.libpath")}</arg> - <arg>-replicationMaxMaps</arg> - <arg>${replicationMaxMaps}</arg> - <arg>-distcpMaxMaps</arg> - <arg>${distcpMaxMaps}</arg> - <arg>-sourceCluster</arg> - <arg>${sourceCluster}</arg> - <arg>-sourceMetastoreUri</arg> - <arg>${sourceMetastoreUri}</arg> - <arg>-sourceHiveServer2Uri</arg> - <arg>${sourceHiveServer2Uri}</arg> - <arg>-sourceDatabase</arg> - <arg>${sourceDatabase}</arg> - <arg>-sourceTable</arg> - <arg>${sourceTable}</arg> - <arg>-sourceStagingPath</arg> - <arg>${sourceStagingPath}</arg> - <arg>-sourceNN</arg> - <arg>${sourceNN}</arg> - <arg>-targetCluster</arg> - <arg>${targetCluster}</arg> - <arg>-targetMetastoreUri</arg> - <arg>${targetMetastoreUri}</arg> - <arg>-targetHiveServer2Uri</arg> - <arg>${targetHiveServer2Uri}</arg> - <arg>-targetStagingPath</arg> - <arg>${targetStagingPath}</arg> - <arg>-targetNN</arg> - <arg>${targetNN}</arg> - <arg>-maxEvents</arg> - <arg>${maxEvents}</arg> - <arg>-distcpMapBandwidth</arg> - <arg>${distcpMapBandwidth}</arg> - <arg>-clusterForJobRun</arg> - <arg>${clusterForJobRun}</arg> - <arg>-clusterForJobRunWriteEP</arg> - <arg>${clusterForJobRunWriteEP}</arg> - <arg>-drJobName</arg> - <arg>${drJobName}-${nominalTime}</arg> - <arg>-executionStage</arg> - <arg>import</arg> - </java> - <ok to="end"/> - <error to="fail"/> - </action> - <kill name="fail"> - <message> - Workflow action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] - </message> - </kill> - <end name="end"/> -</workflow-app> http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties ---------------------------------------------------------------------- diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties deleted file mode 100644 index b14ec7c..0000000 --- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties +++ /dev/null @@ -1,98 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -##### NOTE: This is a TEMPLATE file which can be copied and edited - -##### Recipe properties -falcon.recipe.name=hive-disaster-recovery - - -##### Workflow properties -falcon.recipe.workflow.name=hive-dr-workflow -# Provide Wf absolute path. This can be HDFS or local FS path. If WF is on local FS it will be copied to HDFS -falcon.recipe.workflow.path=/recipes/hive-replication/hive-disaster-recovery-workflow.xml - -##### Cluster properties - -# Change the cluster name where replication job should run here -falcon.recipe.cluster.name=backupCluster -# Change the cluster hdfs write end point here. This is mandatory. -falcon.recipe.cluster.hdfs.writeEndPoint=hdfs://localhost:8020 -# Change the cluster validity start time here -falcon.recipe.cluster.validity.start=2014-10-01T00:00Z -# Change the cluster validity end time here -falcon.recipe.cluster.validity.end=2016-12-30T00:00Z - -##### Scheduling properties - -# Change the process frequency here. Valid frequency type are minutes, hours, days, months -falcon.recipe.process.frequency=minutes(60) - -##### Retry policy properties - -falcon.recipe.retry.policy=periodic -falcon.recipe.retry.delay=minutes(30) -falcon.recipe.retry.attempts=3 -falcon.recipe.retry.onTimeout=false - -##### Tag properties - An optional list of comma separated tags, Key Value Pairs, separated by comma -##### Uncomment to add tags -#falcon.recipe.tags=owner=landing,pipeline=adtech - -##### ACL properties - Uncomment and change ACL if authorization is enabled - -#falcon.recipe.acl.owner=testuser -#falcon.recipe.acl.group=group -#falcon.recipe.acl.permission=0x755 - -##### Custom Job properties - -##### Source Cluster DR properties -sourceCluster=primaryCluster -sourceMetastoreUri=thrift://localhost:9083 -sourceHiveServer2Uri=hive2://localhost:10000 -# For DB level replicaiton to replicate multiple databases specify comma separated list of tables -sourceDatabase=default -# For DB level replication specify * for sourceTable. -# For table level replication to replicate multiple tables specify comma separated list of tables -sourceTable=testtable_dr -## Please specify staging dir in the source without fully qualified domain name. -sourceStagingPath=/apps/hive/tools/dr -sourceNN=hdfs://localhost:8020 - -##### Target Cluster DR properties -targetCluster=backupCluster -targetMetastoreUri=thrift://localhost:9083 -targetHiveServer2Uri=hive2://localhost:10000 -## Please specify staging dir in the target without fully qualified domain name. -targetStagingPath=/apps/hive/tools/dr -targetNN=hdfs://localhost:8020 - -# To ceil the max events processed each time job runs. Set it to max value depending on your bandwidth limit. -# Setting it to -1 will process all the events but can hog up the bandwidth. Use it judiciously! -maxEvents=-1 -# Change it to specify the maximum number of mappers for replication -replicationMaxMaps=5 -# Change it to specify the maximum number of mappers for DistCP -distcpMaxMaps=1 -# Change it to specify the bandwidth in MB for each mapper in DistCP -distcpMapBandwidth=100 - -##### Email Notification for Falcon instance completion -falcon.recipe.notification.type=email -falcon.recipe.notification.receivers=NA \ No newline at end of file http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java b/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java index 41d50df..30a0118 100644 --- a/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java +++ b/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java @@ -40,6 +40,18 @@ public abstract class AbstractCatalogService { public abstract boolean isAlive(Configuration conf, String catalogUrl) throws FalconException; /** + * This method checks if the given db exists in the catalog. + * + * @param conf conf + * @param catalogUrl url for the catalog service + * @param database database the table belongs to + * @return if the db exists + * @throws FalconException exception + */ + public abstract boolean dbExists(Configuration conf, String catalogUrl, + String databaseName) throws FalconException; + + /** * This method checks if the given table exists in the catalog. * * @param conf conf http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java b/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java index 872f91f..4ad62d7 100644 --- a/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java +++ b/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java @@ -208,6 +208,22 @@ public class HiveCatalogService extends AbstractCatalogService { } @Override + public boolean dbExists(Configuration conf, final String catalogUrl, + final String databaseName) throws FalconException { + LOG.info("Checking if the db exists: {}", databaseName); + + try { + HiveMetaStoreClient client = createProxiedClient(conf, catalogUrl); + Database db = client.getDatabase(databaseName); + return db != null; + } catch (NoSuchObjectException e) { + return false; + } catch (Exception e) { + throw new FalconException("Exception checking if the db exists:" + e.getMessage(), e); + } + } + + @Override public boolean tableExists(Configuration conf, final String catalogUrl, final String database, final String tableName) throws FalconException { LOG.info("Checking if the table exists: {}", tableName); http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/common/src/main/resources/startup.properties ---------------------------------------------------------------------- diff --git a/common/src/main/resources/startup.properties b/common/src/main/resources/startup.properties index 87a74bf..dd51963 100644 --- a/common/src/main/resources/startup.properties +++ b/common/src/main/resources/startup.properties @@ -100,6 +100,8 @@ debug.libext.feed.retention.paths=${falcon.libext} debug.libext.feed.replication.paths=${falcon.libext} debug.libext.process.paths=${falcon.libext} +debug.extension.store.uri=file://${user.dir}/target/recipe/store + #Configurations used in ITs it.config.store.uri=file://${user.dir}/target/store it.config.oozie.conf.uri=${user.dir}/target/oozie http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/extensions/pom.xml ---------------------------------------------------------------------- diff --git a/extensions/pom.xml b/extensions/pom.xml new file mode 100644 index 0000000..6a0725a --- /dev/null +++ b/extensions/pom.xml @@ -0,0 +1,112 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.falcon</groupId> + <artifactId>falcon-main</artifactId> + <version>0.10-SNAPSHOT</version> + </parent> + <artifactId>falcon-extensions</artifactId> + <description>Apache Falcon server side extensions Module</description> + <name>Apache Falcon extensions</name> + <packaging>jar</packaging> + + <profiles> + <profile> + <id>hadoop-2</id> + <activation> + <activeByDefault>true</activeByDefault> + </activation> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + </dependency> + </dependencies> + </profile> + </profiles> + + <dependencies> + <dependency> + <groupId>org.apache.falcon</groupId> + <artifactId>falcon-client</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.falcon</groupId> + <artifactId>falcon-common</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-lang3</artifactId> + </dependency> + + <dependency> + <groupId>org.testng</groupId> + <artifactId>testng</artifactId> + </dependency> + + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </dependency> + + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <source>1.7</source> + <target>1.7</target> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>2.4</version> + <configuration> + <excludes> + <exclude>**/log4j.xml</exclude> + </excludes> + </configuration> + <executions> + <execution> + <goals> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> + +</project> http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/extensions/src/main/java/org/apache/falcon/extensions/AbstractExtension.java ---------------------------------------------------------------------- diff --git a/extensions/src/main/java/org/apache/falcon/extensions/AbstractExtension.java b/extensions/src/main/java/org/apache/falcon/extensions/AbstractExtension.java new file mode 100644 index 0000000..11b3725 --- /dev/null +++ b/extensions/src/main/java/org/apache/falcon/extensions/AbstractExtension.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.falcon.extensions; + +import org.apache.falcon.FalconException; +import org.apache.falcon.extensions.mirroring.hdfs.HdfsMirroringExtension; +import org.apache.falcon.extensions.mirroring.hive.HiveMirroringExtension; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; + +/** + * Abstract extension class to be extended by all trusted and custom extensions. + */ +public abstract class AbstractExtension { + private static final List<String> TRUSTED_EXTENSIONS = Arrays.asList( + new HdfsMirroringExtension().getName().toUpperCase(), + new HiveMirroringExtension().getName().toUpperCase()); + private static List<AbstractExtension> extensions = new ArrayList<>(); + + public static List<AbstractExtension> getExtensions() { + if (extensions.isEmpty()) { + extensions.add(new HdfsMirroringExtension()); + extensions.add(new HiveMirroringExtension()); + } + return extensions; + } + + public static boolean isExtensionTrusted(final String extensionName) { + return TRUSTED_EXTENSIONS.contains(extensionName.toUpperCase()); + } + + /* Name cannot be null */ + public abstract String getName(); + + public abstract void validate(final Properties extensionProperties) throws FalconException; + + public abstract Properties getAdditionalProperties(final Properties extensionProperties) throws FalconException; +} + http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/extensions/src/main/java/org/apache/falcon/extensions/Extension.java ---------------------------------------------------------------------- diff --git a/extensions/src/main/java/org/apache/falcon/extensions/Extension.java b/extensions/src/main/java/org/apache/falcon/extensions/Extension.java new file mode 100644 index 0000000..4ab9f5d --- /dev/null +++ b/extensions/src/main/java/org/apache/falcon/extensions/Extension.java @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.falcon.extensions; + +import org.apache.commons.lang3.StringUtils; +import org.apache.falcon.FalconException; +import org.apache.falcon.entity.v0.Entity; +import org.apache.falcon.extensions.store.ExtensionStore; +import org.apache.falcon.extensions.util.ExtensionProcessBuilderUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +/** + * Extension class to construct well formed entities from the templates for trusted extensions. + */ +public class Extension implements ExtensionBuilder { + private static final Logger LOG = LoggerFactory.getLogger(Extension.class); + private static final String EXTENSION_WF_SUFFIX = "-workflow.xml"; + private static final String EXTENSION_TEMPLATE_SUFFIX = "-template.xml"; + + private static void validateProperties(final Properties extensionProperties) throws FalconException { + for (ExtensionProperties option : ExtensionProperties.values()) { + if (extensionProperties.getProperty(option.getName()) == null && option.isRequired()) { + throw new FalconException("Missing extension property: " + option.getName()); + } + } + } + + private static String getExtensionTemplate(final ExtensionStore store, + final Map<String, String> extensionResources, + final String extensionName) throws FalconException { + return store.getExtensionResource(extensionResources.get(extensionName.toLowerCase() + + EXTENSION_TEMPLATE_SUFFIX)); + } + + private static String getWFPath(final Map<String, String> extensionResources, + final String extensionName) { + return extensionResources.get(extensionName.toLowerCase() + EXTENSION_WF_SUFFIX); + } + + @Override + public List<Entity> getEntities(final String extensionName, final Properties extensionProperties) + throws FalconException { + if (StringUtils.isBlank(extensionName)) { + throw new FalconException("Extension name cannot be null or empty"); + } + validateProperties(extensionProperties); + + String name = extensionName.toLowerCase(); + AbstractExtension extension = ExtensionFactory.getExtensionType(name); + if (extension != null) { + extension.validate(extensionProperties); + Properties props = extension.getAdditionalProperties(extensionProperties); + if (props != null && !props.isEmpty()) { + extensionProperties.putAll(props); + } + } + + ExtensionStore store = ExtensionService.getExtensionStore(); + + String resourceName = extensionProperties.getProperty(ExtensionProperties.RESOURCE_NAME.getName()); + if (StringUtils.isBlank(resourceName)) { + resourceName = name; + } + + Map<String, String> extensionResources = store.getExtensionResources(name); + /* Get the resources */ + String extensionTemplate = getExtensionTemplate(store, extensionResources, resourceName); + String wfPath = getWFPath(extensionResources, resourceName); + + /* Get Lib path */ + String wfLibPath = store.getExtensionLibPath(name); + Entity entity = ExtensionProcessBuilderUtils.createProcessFromTemplate(extensionTemplate, + name, extensionProperties, wfPath, wfLibPath); + if (entity == null) { + throw new FalconException("Entity created from the extension template cannot be null"); + } + LOG.info("Extension processing complete"); + return Arrays.asList(entity); + } +} http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/extensions/src/main/java/org/apache/falcon/extensions/ExtensionBuilder.java ---------------------------------------------------------------------- diff --git a/extensions/src/main/java/org/apache/falcon/extensions/ExtensionBuilder.java b/extensions/src/main/java/org/apache/falcon/extensions/ExtensionBuilder.java new file mode 100644 index 0000000..bd52ed4 --- /dev/null +++ b/extensions/src/main/java/org/apache/falcon/extensions/ExtensionBuilder.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.falcon.extensions; + +import org.apache.falcon.FalconException; +import org.apache.falcon.entity.v0.Entity; + +import java.util.Properties; +import java.util.List; + +/** + * Extension interface to be implemented by all trusted and custom extensions. + */ +public interface ExtensionBuilder { + List<Entity> getEntities(final String extensionName, final Properties extensionProperties) throws FalconException; +} http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/extensions/src/main/java/org/apache/falcon/extensions/ExtensionFactory.java ---------------------------------------------------------------------- diff --git a/extensions/src/main/java/org/apache/falcon/extensions/ExtensionFactory.java b/extensions/src/main/java/org/apache/falcon/extensions/ExtensionFactory.java new file mode 100644 index 0000000..41e0648 --- /dev/null +++ b/extensions/src/main/java/org/apache/falcon/extensions/ExtensionFactory.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.falcon.extensions; + +import org.apache.commons.lang3.StringUtils; +import org.apache.falcon.FalconException; + +import java.util.List; + +/** + * Extension Factory class to get the corresponding extension. + */ +public final class ExtensionFactory { + + private ExtensionFactory() { + } + + public static AbstractExtension getExtensionType(final String extensionName) throws FalconException { + if (StringUtils.isBlank(extensionName)) { + return null; + } + + List<AbstractExtension> extensions = AbstractExtension.getExtensions(); + + for (AbstractExtension extension : extensions) { + if (extensionName.equalsIgnoreCase(extension.getName())) { + return extension; + } + } + return null; + } +} http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/extensions/src/main/java/org/apache/falcon/extensions/ExtensionProperties.java ---------------------------------------------------------------------- diff --git a/extensions/src/main/java/org/apache/falcon/extensions/ExtensionProperties.java b/extensions/src/main/java/org/apache/falcon/extensions/ExtensionProperties.java new file mode 100644 index 0000000..b543d73 --- /dev/null +++ b/extensions/src/main/java/org/apache/falcon/extensions/ExtensionProperties.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.falcon.extensions; + +import java.util.Map; +import java.util.HashMap; + +/** + * Extension options. + */ +public enum ExtensionProperties { + JOB_NAME("jobName", "job name"), + RESOURCE_NAME("resourceName", "resource name", false), + //Can't use default as target as extension is generic and need not be used only for replication + CLUSTER_NAME("jobClusterName", "Cluster name where job should run"), + VALIDITY_START("jobValidityStart", "Job validity start"), + VALIDITY_END("jobValidityEnd", "Job validity end"), + FREQUENCY("jobFrequency", "Job frequency"), + TIMEZONE("jobTimezone", "Time zone", false), + // Use defaults for retry + RETRY_POLICY("jobRetryPolicy", "Retry policy", false), + RETRY_DELAY("jobRetryDelay", "Retry delay", false), + RETRY_ATTEMPTS("jobRetryAttempts", "Retry attempts", false), + RETRY_ON_TIMEOUT("jobRetryOnTimeout", "Retry onTimeout", false), + JOB_TAGS("jobTags", "Job tags", false), + JOB_ACL_OWNER("jobAclOwner", "Job acl owner", false), + JOB_ACL_GROUP("jobAclGroup", "Job acl group", false), + JOB_ACL_PERMISSION("jobAclPermission", "Job acl permission", false), + JOB_NOTIFICATION_TYPE("jobNotificationType", "Notification Type", false), + JOB_NOTIFICATION_ADDRESS("jobNotificationReceivers", "Email Notification receivers", false); + + private final String name; + private final String description; + private final boolean isRequired; + + private static Map<String, ExtensionProperties> optionsMap = new HashMap<>(); + static { + for (ExtensionProperties c : ExtensionProperties.values()) { + optionsMap.put(c.getName(), c); + } + } + + public static Map<String, ExtensionProperties> getOptionsMap() { + return optionsMap; + } + + ExtensionProperties(String name, String description) { + this(name, description, true); + } + + ExtensionProperties(String name, String description, boolean isRequired) { + this.name = name; + this.description = description; + this.isRequired = isRequired; + } + + public String getName() { + return this.name; + } + + public String getDescription() { + return description; + } + + public boolean isRequired() { + return isRequired; + } + + @Override + public String toString() { + return getName(); + } +} http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/extensions/src/main/java/org/apache/falcon/extensions/ExtensionService.java ---------------------------------------------------------------------- diff --git a/extensions/src/main/java/org/apache/falcon/extensions/ExtensionService.java b/extensions/src/main/java/org/apache/falcon/extensions/ExtensionService.java new file mode 100644 index 0000000..ffed6ca --- /dev/null +++ b/extensions/src/main/java/org/apache/falcon/extensions/ExtensionService.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.falcon.extensions; + +import org.apache.falcon.FalconException; +import org.apache.falcon.extensions.store.ExtensionStore; +import org.apache.falcon.service.FalconService; + +/** + * Extension service. + */ +public class ExtensionService implements FalconService { + private static ExtensionStore extensionStore = ExtensionStore.get(); + + public static final String SERVICE_NAME = ExtensionService.class.getSimpleName(); + + @Override + public String getName() { + return SERVICE_NAME; + } + + @Override + public void init() throws FalconException { + } + + @Override + public void destroy() throws FalconException { + } + + public static ExtensionStore getExtensionStore() { + return extensionStore; + } +} http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/extensions/src/main/java/org/apache/falcon/extensions/mirroring/hdfs/HdfsMirroringExtension.java ---------------------------------------------------------------------- diff --git a/extensions/src/main/java/org/apache/falcon/extensions/mirroring/hdfs/HdfsMirroringExtension.java b/extensions/src/main/java/org/apache/falcon/extensions/mirroring/hdfs/HdfsMirroringExtension.java new file mode 100644 index 0000000..f1acae2 --- /dev/null +++ b/extensions/src/main/java/org/apache/falcon/extensions/mirroring/hdfs/HdfsMirroringExtension.java @@ -0,0 +1,111 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.falcon.extensions.mirroring.hdfs; + +import org.apache.commons.lang3.StringUtils; +import org.apache.falcon.FalconException; +import org.apache.falcon.entity.ClusterHelper; +import org.apache.falcon.entity.v0.cluster.Cluster; +import org.apache.falcon.extensions.AbstractExtension; + +import java.util.Properties; + +/** + * Hdfs mirroring extension. + */ +public class HdfsMirroringExtension extends AbstractExtension { + + private static final String COMMA_SEPARATOR = ","; + private static final String EXTENSION_NAME = "HDFS-MIRRORING"; + + @Override + public String getName() { + return EXTENSION_NAME; + } + + @Override + public void validate(final Properties extensionProperties) throws FalconException { + for (HdfsMirroringExtensionProperties option : HdfsMirroringExtensionProperties.values()) { + if (extensionProperties.getProperty(option.getName()) == null && option.isRequired()) { + throw new FalconException("Missing extension property: " + option.getName()); + } + } + } + + @Override + public Properties getAdditionalProperties(final Properties extensionProperties) throws FalconException { + Properties additionalProperties = new Properties(); + + // Add default properties if not passed + String distcpMaxMaps = extensionProperties.getProperty(HdfsMirroringExtensionProperties.MAX_MAPS.getName()); + if (StringUtils.isBlank(distcpMaxMaps)) { + additionalProperties.put(HdfsMirroringExtensionProperties.MAX_MAPS.getName(), "1"); + } + + String distcpMapBandwidth = extensionProperties.getProperty( + HdfsMirroringExtensionProperties.MAP_BANDWIDTH_IN_MB.getName()); + if (StringUtils.isBlank(distcpMapBandwidth)) { + additionalProperties.put(HdfsMirroringExtensionProperties.MAP_BANDWIDTH_IN_MB.getName(), "100"); + } + + // Construct fully qualified hdfs src path + String srcPaths = extensionProperties.getProperty(HdfsMirroringExtensionProperties + .SOURCE_DIR.getName()); + StringBuilder absoluteSrcPaths = new StringBuilder(); + String sourceClusterName = extensionProperties.getProperty( + HdfsMirroringExtensionProperties.SOURCE_CLUSTER.getName()); + + // Since source cluster get read interface + Cluster srcCluster = ClusterHelper.getCluster(sourceClusterName); + if (srcCluster == null) { + throw new FalconException("Cluster entity " + sourceClusterName + " not found"); + } + String srcClusterEndPoint = ClusterHelper.getReadOnlyStorageUrl(srcCluster); + + if (StringUtils.isNotBlank(srcPaths)) { + String[] paths = srcPaths.split(COMMA_SEPARATOR); + + for (String path : paths) { + StringBuilder srcpath = new StringBuilder(srcClusterEndPoint); + srcpath.append(path.trim()); + srcpath.append(COMMA_SEPARATOR); + absoluteSrcPaths.append(srcpath); + } + } + + additionalProperties.put(HdfsMirroringExtensionProperties.SOURCE_DIR.getName(), + StringUtils.removeEnd(absoluteSrcPaths.toString(), COMMA_SEPARATOR)); + + // add sourceClusterFS and targetClusterFS + additionalProperties.put(HdfsMirroringExtensionProperties.SOURCE_CLUSTER_FS_WRITE_ENDPOINT.getName(), + ClusterHelper.getStorageUrl(srcCluster)); + + String targetClusterName = extensionProperties.getProperty( + HdfsMirroringExtensionProperties.TARGET_CLUSTER.getName()); + + Cluster targetCluster = ClusterHelper.getCluster(targetClusterName); + if (targetCluster == null) { + throw new FalconException("Cluster entity " + targetClusterName + " not found"); + } + additionalProperties.put(HdfsMirroringExtensionProperties.TARGET_CLUSTER_FS_WRITE_ENDPOINT.getName(), + ClusterHelper.getStorageUrl(targetCluster)); + return additionalProperties; + } + +} http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/extensions/src/main/java/org/apache/falcon/extensions/mirroring/hdfs/HdfsMirroringExtensionProperties.java ---------------------------------------------------------------------- diff --git a/extensions/src/main/java/org/apache/falcon/extensions/mirroring/hdfs/HdfsMirroringExtensionProperties.java b/extensions/src/main/java/org/apache/falcon/extensions/mirroring/hdfs/HdfsMirroringExtensionProperties.java new file mode 100644 index 0000000..7d24b45 --- /dev/null +++ b/extensions/src/main/java/org/apache/falcon/extensions/mirroring/hdfs/HdfsMirroringExtensionProperties.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.falcon.extensions.mirroring.hdfs; + +/** + * Hdfs Extension properties. + */ +public enum HdfsMirroringExtensionProperties { + SOURCE_DIR("sourceDir", "Location of source data to replicate"), + SOURCE_CLUSTER("sourceCluster", "Source cluster"), + SOURCE_CLUSTER_FS_WRITE_ENDPOINT("sourceClusterFS", "Source cluster end point", false), + TARGET_DIR("targetDir", "Location on target cluster for replication"), + TARGET_CLUSTER("targetCluster", "Target cluster"), + TARGET_CLUSTER_FS_WRITE_ENDPOINT("targetClusterFS", "Target cluster end point", false), + MAX_MAPS("distcpMaxMaps", "Maximum number of maps used during replication", false), + MAP_BANDWIDTH_IN_MB("distcpMapBandwidth", "Bandwidth in MB/s used by each mapper during replication", + false); + + private final String name; + private final String description; + private final boolean isRequired; + + HdfsMirroringExtensionProperties(String name, String description) { + this(name, description, true); + } + + HdfsMirroringExtensionProperties(String name, String description, boolean isRequired) { + this.name = name; + this.description = description; + this.isRequired = isRequired; + } + + public String getName() { + return this.name; + } + + public String getDescription() { + return description; + } + + public boolean isRequired() { + return isRequired; + } + + @Override + public String toString() { + return getName(); + } +}
