Repository: falcon Updated Branches: refs/heads/asf-site 8609ffd6f -> 6f5b476cc
http://git-wip-us.apache.org/repos/asf/falcon/blob/6f5b476c/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties ---------------------------------------------------------------------- diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties deleted file mode 100644 index 8d00bb5..0000000 --- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties +++ /dev/null @@ -1,108 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -##### NOTE: This is a TEMPLATE file which can be copied and edited - -##### Recipe properties -falcon.recipe.name=hive-disaster-recovery - - -##### Workflow properties -falcon.recipe.workflow.name=hive-dr-workflow -# Provide Wf absolute path. This can be HDFS or local FS path. If WF is on local FS it will be copied to HDFS -falcon.recipe.workflow.path=/recipes/hive-replication/hive-disaster-recovery-secure-workflow.xml - -##### Cluster properties - -# Change the cluster name where replication job should run here -falcon.recipe.cluster.name=backupCluster -# Change the cluster hdfs write end point here. This is mandatory. -falcon.recipe.cluster.hdfs.writeEndPoint=hdfs://localhost:8020 -# Change the cluster validity start time here -falcon.recipe.cluster.validity.start=2014-10-01T00:00Z -# Change the cluster validity end time here -falcon.recipe.cluster.validity.end=2016-12-30T00:00Z -# Change the cluster namenode kerberos principal. This is mandatory on secure clusters. -falcon.recipe.nn.principal=nn/[email protected] - -##### Scheduling properties - -# Change the process frequency here. Valid frequency type are minutes, hours, days, months -falcon.recipe.process.frequency=minutes(60) - -##### Retry policy properties - -falcon.recipe.retry.policy=periodic -falcon.recipe.retry.delay=minutes(30) -falcon.recipe.retry.attempts=3 -falcon.recipe.retry.onTimeout=false - -##### Tag properties - An optional list of comma separated tags, Key Value Pairs, separated by comma -##### Uncomment to add tags -#falcon.recipe.tags=owner=landing,pipeline=adtech - -##### ACL properties - Uncomment and change ACL if authorization is enabled - -#falcon.recipe.acl.owner=testuser -#falcon.recipe.acl.group=group -#falcon.recipe.acl.permission=0x755 - -##### Custom Job properties - -##### Source Cluster DR properties -sourceCluster=primaryCluster -sourceMetastoreUri=thrift://localhost:9083 -sourceHiveServer2Uri=hive2://localhost:10000 -# For DB level replicaiton to replicate multiple databases specify comma separated list of tables -sourceDatabase=default -# For DB level replication specify * for sourceTable. -# For table level replication to replicate multiple tables specify comma separated list of tables -sourceTable=testtable_dr -## Please specify staging dir in the source without fully qualified domain name. -sourceStagingPath=/apps/hive/tools/dr -sourceNN=hdfs://localhost:8020 -# Specify kerberos principal required to access source namenode and hive servers, optional on non-secure cluster. -sourceNNKerberosPrincipal=nn/[email protected] -sourceHiveMetastoreKerberosPrincipal=hive/[email protected] -sourceHive2KerberosPrincipal=hive/[email protected] - -##### Target Cluster DR properties -targetCluster=backupCluster -targetMetastoreUri=thrift://localhost:9083 -targetHiveServer2Uri=hive2://localhost:10000 -## Please specify staging dir in the target without fully qualified domain name. -targetStagingPath=/apps/hive/tools/dr -targetNN=hdfs://localhost:8020 -# Specify kerberos principal required to access target namenode and hive servers, optional on non-secure cluster. -targetNNKerberosPrincipal=nn/[email protected] -targetHiveMetastoreKerberosPrincipal=hive/[email protected] -targetHive2KerberosPrincipal=hive/[email protected] - -# To ceil the max events processed each time job runs. Set it to max value depending on your bandwidth limit. -# Setting it to -1 will process all the events but can hog up the bandwidth. Use it judiciously! -maxEvents=-1 -# Change it to specify the maximum number of mappers for replication -replicationMaxMaps=5 -# Change it to specify the maximum number of mappers for DistCP -distcpMaxMaps=1 -# Change it to specify the bandwidth in MB for each mapper in DistCP -distcpMapBandwidth=100 - -##### Email Notification for Falcon instance completion -falcon.recipe.notification.type=email -falcon.recipe.notification.receivers=NA \ No newline at end of file http://git-wip-us.apache.org/repos/asf/falcon/blob/6f5b476c/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-template.xml ---------------------------------------------------------------------- diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-template.xml b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-template.xml deleted file mode 100644 index f0de091..0000000 --- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-template.xml +++ /dev/null @@ -1,45 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> - -<process name="##name##" xmlns="uri:falcon:process:0.1"> - <clusters> - <!-- source --> - <cluster name="##cluster.name##"> - <validity end="##cluster.validity.end##" start="##cluster.validity.start##"/> - </cluster> - </clusters> - - <tags>_falcon_mirroring_type=HIVE</tags> - - <parallel>1</parallel> - <!-- Replication needs to run only once to catch up --> - <order>LAST_ONLY</order> - <frequency>##process.frequency##</frequency> - <timezone>UTC</timezone> - - <properties> - <property name="oozie.wf.subworkflow.classpath.inheritance" value="true"/> - </properties> - - <workflow name="##workflow.name##" engine="oozie" - path="/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml" lib="##workflow.lib.path##"/> - <retry policy="##retry.policy##" delay="##retry.delay##" attempts="3"/> - <notification type="##notification.type##" to="##notification.receivers##"/> - <ACL/> -</process> http://git-wip-us.apache.org/repos/asf/falcon/blob/6f5b476c/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml ---------------------------------------------------------------------- diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml deleted file mode 100644 index 296e049..0000000 --- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml +++ /dev/null @@ -1,249 +0,0 @@ -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> -<workflow-app xmlns='uri:oozie:workflow:0.3' name='falcon-dr-hive-workflow'> - <start to='last-event'/> - <action name="last-event"> - <java> - <job-tracker>${jobTracker}</job-tracker> - <name-node>${nameNode}</name-node> - <configuration> - <property> <!-- hadoop 2 parameter --> - <name>oozie.launcher.mapreduce.job.user.classpath.first</name> - <value>true</value> - </property> - <property> - <name>mapred.job.queue.name</name> - <value>${queueName}</value> - </property> - <property> - <name>oozie.launcher.mapred.job.priority</name> - <value>${jobPriority}</value> - </property> - <property> - <name>oozie.use.system.libpath</name> - <value>true</value> - </property> - <property> - <name>oozie.action.sharelib.for.java</name> - <value>distcp,hive,hive2,hcatalog</value> - </property> - </configuration> - <main-class>org.apache.falcon.hive.HiveDRTool</main-class> - <arg>-Dmapred.job.queue.name=${queueName}</arg> - <arg>-Dmapred.job.priority=${jobPriority}</arg> - <arg>-falconLibPath</arg> - <arg>${wf:conf("falcon.libpath")}</arg> - <arg>-sourceCluster</arg> - <arg>${sourceCluster}</arg> - <arg>-sourceMetastoreUri</arg> - <arg>${sourceMetastoreUri}</arg> - <arg>-sourceHiveServer2Uri</arg> - <arg>${sourceHiveServer2Uri}</arg> - <arg>-sourceDatabase</arg> - <arg>${sourceDatabase}</arg> - <arg>-sourceTable</arg> - <arg>${sourceTable}</arg> - <arg>-sourceStagingPath</arg> - <arg>${sourceStagingPath}</arg> - <arg>-sourceNN</arg> - <arg>${sourceNN}</arg> - <arg>-targetCluster</arg> - <arg>${targetCluster}</arg> - <arg>-targetMetastoreUri</arg> - <arg>${targetMetastoreUri}</arg> - <arg>-targetHiveServer2Uri</arg> - <arg>${targetHiveServer2Uri}</arg> - <arg>-targetStagingPath</arg> - <arg>${targetStagingPath}</arg> - <arg>-targetNN</arg> - <arg>${targetNN}</arg> - <arg>-maxEvents</arg> - <arg>${maxEvents}</arg> - <arg>-clusterForJobRun</arg> - <arg>${clusterForJobRun}</arg> - <arg>-clusterForJobRunWriteEP</arg> - <arg>${clusterForJobRunWriteEP}</arg> - <arg>-drJobName</arg> - <arg>${drJobName}-${nominalTime}</arg> - <arg>-executionStage</arg> - <arg>lastevents</arg> - </java> - <ok to="export-dr-replication"/> - <error to="fail"/> - </action> - <!-- Export Replication action --> - <action name="export-dr-replication"> - <java> - <job-tracker>${jobTracker}</job-tracker> - <name-node>${nameNode}</name-node> - <configuration> - <property> <!-- hadoop 2 parameter --> - <name>oozie.launcher.mapreduce.job.user.classpath.first</name> - <value>true</value> - </property> - <property> - <name>mapred.job.queue.name</name> - <value>${queueName}</value> - </property> - <property> - <name>oozie.launcher.mapred.job.priority</name> - <value>${jobPriority}</value> - </property> - <property> - <name>oozie.use.system.libpath</name> - <value>true</value> - </property> - <property> - <name>oozie.action.sharelib.for.java</name> - <value>distcp,hive,hive2,hcatalog</value> - </property> - </configuration> - <main-class>org.apache.falcon.hive.HiveDRTool</main-class> - <arg>-Dmapred.job.queue.name=${queueName}</arg> - <arg>-Dmapred.job.priority=${jobPriority}</arg> - <arg>-falconLibPath</arg> - <arg>${wf:conf("falcon.libpath")}</arg> - <arg>-replicationMaxMaps</arg> - <arg>${replicationMaxMaps}</arg> - <arg>-distcpMaxMaps</arg> - <arg>${distcpMaxMaps}</arg> - <arg>-sourceCluster</arg> - <arg>${sourceCluster}</arg> - <arg>-sourceMetastoreUri</arg> - <arg>${sourceMetastoreUri}</arg> - <arg>-sourceHiveServer2Uri</arg> - <arg>${sourceHiveServer2Uri}</arg> - <arg>-sourceDatabase</arg> - <arg>${sourceDatabase}</arg> - <arg>-sourceTable</arg> - <arg>${sourceTable}</arg> - <arg>-sourceStagingPath</arg> - <arg>${sourceStagingPath}</arg> - <arg>-sourceNN</arg> - <arg>${sourceNN}</arg> - <arg>-targetCluster</arg> - <arg>${targetCluster}</arg> - <arg>-targetMetastoreUri</arg> - <arg>${targetMetastoreUri}</arg> - <arg>-targetHiveServer2Uri</arg> - <arg>${targetHiveServer2Uri}</arg> - <arg>-targetStagingPath</arg> - <arg>${targetStagingPath}</arg> - <arg>-targetNN</arg> - <arg>${targetNN}</arg> - <arg>-maxEvents</arg> - <arg>${maxEvents}</arg> - <arg>-distcpMapBandwidth</arg> - <arg>${distcpMapBandwidth}</arg> - <arg>-clusterForJobRun</arg> - <arg>${clusterForJobRun}</arg> - <arg>-clusterForJobRunWriteEP</arg> - <arg>${clusterForJobRunWriteEP}</arg> - <arg>-drJobName</arg> - <arg>${drJobName}-${nominalTime}</arg> - <arg>-executionStage</arg> - <arg>export</arg> - <arg>-counterLogDir</arg> - <arg>${logDir}/job-${nominalTime}/${srcClusterName == 'NA' ? '' : srcClusterName}/</arg> - </java> - <ok to="import-dr-replication"/> - <error to="fail"/> - </action> - <!-- Import Replication action --> - <action name="import-dr-replication"> - <java> - <job-tracker>${jobTracker}</job-tracker> - <name-node>${nameNode}</name-node> - <configuration> - <property> <!-- hadoop 2 parameter --> - <name>oozie.launcher.mapreduce.job.user.classpath.first</name> - <value>true</value> - </property> - <property> - <name>mapred.job.queue.name</name> - <value>${queueName}</value> - </property> - <property> - <name>oozie.launcher.mapred.job.priority</name> - <value>${jobPriority}</value> - </property> - <property> - <name>oozie.use.system.libpath</name> - <value>true</value> - </property> - <property> - <name>oozie.action.sharelib.for.java</name> - <value>distcp,hive,hive2,hcatalog</value> - </property> - </configuration> - <main-class>org.apache.falcon.hive.HiveDRTool</main-class> - <arg>-Dmapred.job.queue.name=${queueName}</arg> - <arg>-Dmapred.job.priority=${jobPriority}</arg> - <arg>-falconLibPath</arg> - <arg>${wf:conf("falcon.libpath")}</arg> - <arg>-replicationMaxMaps</arg> - <arg>${replicationMaxMaps}</arg> - <arg>-distcpMaxMaps</arg> - <arg>${distcpMaxMaps}</arg> - <arg>-sourceCluster</arg> - <arg>${sourceCluster}</arg> - <arg>-sourceMetastoreUri</arg> - <arg>${sourceMetastoreUri}</arg> - <arg>-sourceHiveServer2Uri</arg> - <arg>${sourceHiveServer2Uri}</arg> - <arg>-sourceDatabase</arg> - <arg>${sourceDatabase}</arg> - <arg>-sourceTable</arg> - <arg>${sourceTable}</arg> - <arg>-sourceStagingPath</arg> - <arg>${sourceStagingPath}</arg> - <arg>-sourceNN</arg> - <arg>${sourceNN}</arg> - <arg>-targetCluster</arg> - <arg>${targetCluster}</arg> - <arg>-targetMetastoreUri</arg> - <arg>${targetMetastoreUri}</arg> - <arg>-targetHiveServer2Uri</arg> - <arg>${targetHiveServer2Uri}</arg> - <arg>-targetStagingPath</arg> - <arg>${targetStagingPath}</arg> - <arg>-targetNN</arg> - <arg>${targetNN}</arg> - <arg>-maxEvents</arg> - <arg>${maxEvents}</arg> - <arg>-distcpMapBandwidth</arg> - <arg>${distcpMapBandwidth}</arg> - <arg>-clusterForJobRun</arg> - <arg>${clusterForJobRun}</arg> - <arg>-clusterForJobRunWriteEP</arg> - <arg>${clusterForJobRunWriteEP}</arg> - <arg>-drJobName</arg> - <arg>${drJobName}-${nominalTime}</arg> - <arg>-executionStage</arg> - <arg>import</arg> - </java> - <ok to="end"/> - <error to="fail"/> - </action> - <kill name="fail"> - <message> - Workflow action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] - </message> - </kill> - <end name="end"/> -</workflow-app> http://git-wip-us.apache.org/repos/asf/falcon/blob/6f5b476c/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties ---------------------------------------------------------------------- diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties deleted file mode 100644 index b14ec7c..0000000 --- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties +++ /dev/null @@ -1,98 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -##### NOTE: This is a TEMPLATE file which can be copied and edited - -##### Recipe properties -falcon.recipe.name=hive-disaster-recovery - - -##### Workflow properties -falcon.recipe.workflow.name=hive-dr-workflow -# Provide Wf absolute path. This can be HDFS or local FS path. If WF is on local FS it will be copied to HDFS -falcon.recipe.workflow.path=/recipes/hive-replication/hive-disaster-recovery-workflow.xml - -##### Cluster properties - -# Change the cluster name where replication job should run here -falcon.recipe.cluster.name=backupCluster -# Change the cluster hdfs write end point here. This is mandatory. -falcon.recipe.cluster.hdfs.writeEndPoint=hdfs://localhost:8020 -# Change the cluster validity start time here -falcon.recipe.cluster.validity.start=2014-10-01T00:00Z -# Change the cluster validity end time here -falcon.recipe.cluster.validity.end=2016-12-30T00:00Z - -##### Scheduling properties - -# Change the process frequency here. Valid frequency type are minutes, hours, days, months -falcon.recipe.process.frequency=minutes(60) - -##### Retry policy properties - -falcon.recipe.retry.policy=periodic -falcon.recipe.retry.delay=minutes(30) -falcon.recipe.retry.attempts=3 -falcon.recipe.retry.onTimeout=false - -##### Tag properties - An optional list of comma separated tags, Key Value Pairs, separated by comma -##### Uncomment to add tags -#falcon.recipe.tags=owner=landing,pipeline=adtech - -##### ACL properties - Uncomment and change ACL if authorization is enabled - -#falcon.recipe.acl.owner=testuser -#falcon.recipe.acl.group=group -#falcon.recipe.acl.permission=0x755 - -##### Custom Job properties - -##### Source Cluster DR properties -sourceCluster=primaryCluster -sourceMetastoreUri=thrift://localhost:9083 -sourceHiveServer2Uri=hive2://localhost:10000 -# For DB level replicaiton to replicate multiple databases specify comma separated list of tables -sourceDatabase=default -# For DB level replication specify * for sourceTable. -# For table level replication to replicate multiple tables specify comma separated list of tables -sourceTable=testtable_dr -## Please specify staging dir in the source without fully qualified domain name. -sourceStagingPath=/apps/hive/tools/dr -sourceNN=hdfs://localhost:8020 - -##### Target Cluster DR properties -targetCluster=backupCluster -targetMetastoreUri=thrift://localhost:9083 -targetHiveServer2Uri=hive2://localhost:10000 -## Please specify staging dir in the target without fully qualified domain name. -targetStagingPath=/apps/hive/tools/dr -targetNN=hdfs://localhost:8020 - -# To ceil the max events processed each time job runs. Set it to max value depending on your bandwidth limit. -# Setting it to -1 will process all the events but can hog up the bandwidth. Use it judiciously! -maxEvents=-1 -# Change it to specify the maximum number of mappers for replication -replicationMaxMaps=5 -# Change it to specify the maximum number of mappers for DistCP -distcpMaxMaps=1 -# Change it to specify the bandwidth in MB for each mapper in DistCP -distcpMapBandwidth=100 - -##### Email Notification for Falcon instance completion -falcon.recipe.notification.type=email -falcon.recipe.notification.receivers=NA \ No newline at end of file
