Repository: falcon Updated Branches: refs/heads/master 3435e2096 -> bd32b610e
FALCON-1914 Hive mirroring should work for secure to unsecure & viceversa. Also cleanup artifacts. Author: Sowmya Ramesh <[email protected]> Reviewers: "Balu Vellanki <[email protected]>" Closes #205 from sowmyaramesh/FALCON-1914 Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/bd32b610 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/bd32b610 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/bd32b610 Branch: refs/heads/master Commit: bd32b610e617ae05027b45e3a5c7672613afdfdc Parents: 3435e20 Author: Sowmya Ramesh <[email protected]> Authored: Tue Jul 5 15:25:19 2016 -0700 Committer: Sowmya Ramesh <[email protected]> Committed: Tue Jul 5 15:25:19 2016 -0700 ---------------------------------------------------------------------- .../main/META/hive-mirroring-properties.json | 16 +- .../META/hive-mirroring-secure-properties.json | 191 ------------------- .../runtime/hive-mirroring-secure-template.xml | 45 ----- .../org/apache/falcon/extensions/Extension.java | 2 +- .../mirroring/hive/HiveMirroringExtension.java | 64 ++++--- 5 files changed, 52 insertions(+), 266 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/bd32b610/addons/extensions/hive-mirroring/src/main/META/hive-mirroring-properties.json ---------------------------------------------------------------------- diff --git a/addons/extensions/hive-mirroring/src/main/META/hive-mirroring-properties.json b/addons/extensions/hive-mirroring/src/main/META/hive-mirroring-properties.json index a9f3d1b..e019e68 100644 --- a/addons/extensions/hive-mirroring/src/main/META/hive-mirroring-properties.json +++ b/addons/extensions/hive-mirroring/src/main/META/hive-mirroring-properties.json @@ -1,5 +1,5 @@ { - "shortDescription":"This extension implements replicating hive metadata and data from one Hadoop cluster to another Hadoop cluster.", + "shortDescription": "This extension implements replicating hive metadata and data from one Hadoop cluster to another Hadoop cluster.", "properties":[ { "propertyName":"jobName", @@ -116,6 +116,12 @@ "example":"/apps/hive/dr" }, { + "propertyName":"sourceHive2KerberosPrincipal", + "required":true, + "description":"Required on secure clusters. Kerberos principal required to access hive servers ", + "example":"hive/[email protected]" + }, + { "propertyName":"targetCluster", "required":true, "description":"target cluster for hive mirroring", @@ -134,6 +140,12 @@ "example":"/apps/hive/dr" }, { + "propertyName":"targetHive2KerberosPrincipal", + "required":true, + "description":"Required on secure clusters. Kerberos principal required to access hive servers ", + "example":"hive/[email protected]" + }, + { "propertyName":"maxEvents", "required":false, "description":"To ceil the max events processed each time the job runs. Set it to max value depending on your bandwidth limit. Setting it to -1 will process all the events but can hog up the bandwidth. Use it judiciously!", @@ -176,4 +188,4 @@ "example":"[email protected], [email protected]" } ] -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/falcon/blob/bd32b610/addons/extensions/hive-mirroring/src/main/META/hive-mirroring-secure-properties.json ---------------------------------------------------------------------- diff --git a/addons/extensions/hive-mirroring/src/main/META/hive-mirroring-secure-properties.json b/addons/extensions/hive-mirroring/src/main/META/hive-mirroring-secure-properties.json deleted file mode 100644 index 8ec03b5..0000000 --- a/addons/extensions/hive-mirroring/src/main/META/hive-mirroring-secure-properties.json +++ /dev/null @@ -1,191 +0,0 @@ -{ - "shortDescription": "This extension implements replicating hive metadata and data from one Hadoop cluster to another Hadoop cluster in secure environment.", - "properties":[ - { - "propertyName":"jobName", - "required":true, - "description":"Unique job name", - "example":"hive-monthly-sales-dr" - }, - { - "propertyName":"jobClusterName", - "required":true, - "description":"Cluster where job should run", - "example":"backupCluster" - }, - { - "propertyName":"jobValidityStart", - "required":true, - "description":"Job validity start time", - "example":"2016-03-03T00:00Z" - }, - { - "propertyName":"jobValidityEnd", - "required":true, - "description":"Job validity end time", - "example":"2018-03-13T00:00Z" - }, - { - "propertyName":"jobFrequency", - "required":true, - "description":"job frequency. Valid frequency types are minutes, hours, days, months", - "example":"months(1)" - }, - { - "propertyName":"jobTimezone", - "required":false, - "description":"Time zone for the job", - "example":"GMT" - }, - { - "propertyName":"jobTags", - "required":false, - "description":"list of comma separated tags. Key Value Pairs, separated by comma", - "example":"[email protected], [email protected], _department_type=forecasting" - }, - { - "propertyName":"jobRetryPolicy", - "required":false, - "description":"Job retry policy", - "example":"periodic" - }, - { - "propertyName":"jobRetryDelay", - "required":false, - "description":"Job retry delay", - "example":"minutes(30)" - }, - { - "propertyName":"jobRetryAttempts", - "required":false, - "description":"Job retry attempts", - "example":"3" - }, - { - "propertyName":"jobRetryOnTimeout", - "required":false, - "description":"Job retry on timeout", - "example":true - }, - { - "propertyName":"jobAclOwner", - "required":false, - "description":"ACL owner", - "example":"ambari-qa" - }, - { - "propertyName":"jobAclGroup", - "required":false, - "description":"ACL group", - "example":"users" - }, - { - "propertyName":"jobAclPermission", - "required":false, - "description":"ACL permission", - "example":"0x755" - }, - { - "propertyName":"sourceCluster", - "required":true, - "description":"Source cluster for hive mirroring", - "example":"primaryCluster" - }, - { - "propertyName":"sourceHiveServer2Uri", - "required":true, - "description":"Hive2 server end point", - "example":"hive2://localhost:10000" - }, - { - "propertyName":"sourceDatabases", - "required":true, - "description":"For DB level replication specify multiple comma separated databases to replicate", - "example":"salesDb" - }, - { - "propertyName":"sourceTables", - "required":false, - "description":"For table level replication specify multiple comma separated tables to replicate", - "example":"monthly_sales1, monthly_sales2" - }, - { - "propertyName":"sourceStagingPath", - "required":false, - "description":"Staging path on source", - "example":"/apps/hive/dr" - }, - { - "propertyName":"sourceHive2KerberosPrincipal", - "required":true, - "description":"Required on secure clusters. Kerberos principal required to access hive servers ", - "example":"hive/[email protected]" - }, - { - "propertyName":"targetCluster", - "required":true, - "description":"target cluster for hive mirroring", - "example":"backupCluster" - }, - { - "propertyName":"targetHiveServer2Uri", - "required":true, - "description":"Hive2 server end point", - "example":"hive2://localhost:10000" - }, - { - "propertyName":"targetStagingPath", - "required":false, - "description":"Staging path on target", - "example":"/apps/hive/dr" - }, - { - "propertyName":"targetHive2KerberosPrincipal", - "required":true, - "description":"Required on secure clusters. Kerberos principal required to access hive servers ", - "example":"hive/[email protected]" - }, - { - "propertyName":"maxEvents", - "required":false, - "description":"To ceil the max events processed each time the job runs. Set it to max value depending on your bandwidth limit. Setting it to -1 will process all the events but can hog up the bandwidth. Use it judiciously!", - "example":"10000" - }, - { - "propertyName":"replicationMaxMaps", - "required":false, - "description":"Maximum number of mappers to use for hive replication", - "example":"1" - }, - { - "propertyName":"distcpMaxMaps", - "required":false, - "description":"Maximum number of mappers for DistCP", - "example":"1" - }, - { - "propertyName":"distcpMapBandwidth", - "required":false, - "description":"Bandwidth in MB for each mapper in DistCP", - "example":"100" - }, - { - "propertyName":"tdeEncryptionEnabled", - "required":false, - "description":"Set this flag to true if TDE encryption is enabled on source and target. Default value is false", - "example":"true" - }, - { - "propertyName":"jobNotificationType", - "required":false, - "description":"Email Notification for Falcon instance completion", - "example":"email" - }, - { - "propertyName":"jobNotificationReceivers", - "required":false, - "description":"Comma separated email Id's", - "example":"[email protected], [email protected]" - } - ] -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/falcon/blob/bd32b610/addons/extensions/hive-mirroring/src/main/resources/runtime/hive-mirroring-secure-template.xml ---------------------------------------------------------------------- diff --git a/addons/extensions/hive-mirroring/src/main/resources/runtime/hive-mirroring-secure-template.xml b/addons/extensions/hive-mirroring/src/main/resources/runtime/hive-mirroring-secure-template.xml deleted file mode 100644 index 4497bb4..0000000 --- a/addons/extensions/hive-mirroring/src/main/resources/runtime/hive-mirroring-secure-template.xml +++ /dev/null @@ -1,45 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> - -<process name="##jobName##" xmlns="uri:falcon:process:0.1"> - <clusters> - <!-- source --> - <cluster name="##jobClusterName##"> - <validity end="##jobValidityEnd##" start="##jobValidityStart##"/> - </cluster> - </clusters> - - <tags/> - - <parallel>1</parallel> - <!-- Replication needs to run only once to catch up --> - <order>LAST_ONLY</order> - <frequency>##jobFrequency##</frequency> - <timezone>##jobTimezone##</timezone> - - <properties> - <property name="oozie.wf.subworkflow.classpath.inheritance" value="true"/> - </properties> - - <workflow name="##jobWorkflowName##" engine="##jobWorkflowEngine##" - path="##jobWorkflowPath##" lib="##jobWorkflowLibPath##"/> - <retry policy="##jobRetryPolicy##" delay="##jobRetryDelay##" attempts="3"/> - <notification type="##jobNotificationType##" to="##jobNotificationReceivers##"/> - <ACL/> -</process> http://git-wip-us.apache.org/repos/asf/falcon/blob/bd32b610/extensions/src/main/java/org/apache/falcon/extensions/Extension.java ---------------------------------------------------------------------- diff --git a/extensions/src/main/java/org/apache/falcon/extensions/Extension.java b/extensions/src/main/java/org/apache/falcon/extensions/Extension.java index 4ab9f5d..6c83fe8 100644 --- a/extensions/src/main/java/org/apache/falcon/extensions/Extension.java +++ b/extensions/src/main/java/org/apache/falcon/extensions/Extension.java @@ -86,7 +86,7 @@ public class Extension implements ExtensionBuilder { Map<String, String> extensionResources = store.getExtensionResources(name); /* Get the resources */ - String extensionTemplate = getExtensionTemplate(store, extensionResources, resourceName); + String extensionTemplate = getExtensionTemplate(store, extensionResources, name); String wfPath = getWFPath(extensionResources, resourceName); /* Get Lib path */ http://git-wip-us.apache.org/repos/asf/falcon/blob/bd32b610/extensions/src/main/java/org/apache/falcon/extensions/mirroring/hive/HiveMirroringExtension.java ---------------------------------------------------------------------- diff --git a/extensions/src/main/java/org/apache/falcon/extensions/mirroring/hive/HiveMirroringExtension.java b/extensions/src/main/java/org/apache/falcon/extensions/mirroring/hive/HiveMirroringExtension.java index 9222e0a..c3bd7a7 100644 --- a/extensions/src/main/java/org/apache/falcon/extensions/mirroring/hive/HiveMirroringExtension.java +++ b/extensions/src/main/java/org/apache/falcon/extensions/mirroring/hive/HiveMirroringExtension.java @@ -139,13 +139,18 @@ public class HiveMirroringExtension extends AbstractExtension { } additionalProperties.put(HiveMirroringExtensionProperties.CLUSTER_FOR_JOB_RUN_WRITE_EP.getName(), ClusterHelper.getStorageUrl(jobCluster)); - if (SecurityUtil.isSecurityEnabled()) { + + // Check if job cluster is secure + String jobClusterKerberosPrincipal = ClusterHelper.getPropertyValue(jobCluster, SecurityUtil.NN_PRINCIPAL); + if (StringUtils.isNotBlank(jobClusterKerberosPrincipal)) { // Add -secure and update the resource name String resourceName = getName().toLowerCase() + SECURE_RESOURCE; additionalProperties.put(ExtensionProperties.RESOURCE_NAME.getName(), resourceName); - additionalProperties.put(HiveMirroringExtensionProperties.CLUSTER_FOR_JOB_NN_KERBEROS_PRINCIPAL.getName(), - ClusterHelper.getPropertyValue(jobCluster, SecurityUtil.NN_PRINCIPAL)); + } else { + jobClusterKerberosPrincipal = NOT_APPLICABLE; } + additionalProperties.put(HiveMirroringExtensionProperties.CLUSTER_FOR_JOB_NN_KERBEROS_PRINCIPAL.getName(), + jobClusterKerberosPrincipal); // Properties for src cluster String srcClusterName = extensionProperties.getProperty(HiveMirroringExtensionProperties.SOURCE_CLUSTER @@ -165,20 +170,22 @@ public class HiveMirroringExtension extends AbstractExtension { additionalProperties.put(HiveMirroringExtensionProperties.SOURCE_TABLES.getName(), ALL_TABLES); } - if (SecurityUtil.isSecurityEnabled()) { - String hive2Principal = extensionProperties.getProperty(HiveMirroringExtensionProperties - .SOURCE_HIVE2_KERBEROS_PRINCIPAL.getName()); - if (StringUtils.isBlank(hive2Principal)) { - throw new FalconException("Hive server2 kerberos principal for cluster " + srcCluster.getName() - + "not passed for extension " + jobName); - } + // Check if source cluster is secure + String srcClusterKerberosPrincipal = ClusterHelper.getPropertyValue(srcCluster, SecurityUtil.NN_PRINCIPAL); + if (StringUtils.isBlank(srcClusterKerberosPrincipal)) { + srcClusterKerberosPrincipal = NOT_APPLICABLE; + } - additionalProperties.put(HiveMirroringExtensionProperties.SOURCE_NN_KERBEROS_PRINCIPAL.getName(), - ClusterHelper.getPropertyValue(srcCluster, SecurityUtil.NN_PRINCIPAL)); - additionalProperties.put( - HiveMirroringExtensionProperties.SOURCE_HIVE_METASTORE_KERBEROS_PRINCIPAL.getName(), - ClusterHelper.getPropertyValue(srcCluster, SecurityUtil.HIVE_METASTORE_KERBEROS_PRINCIPAL)); + String srcHiveMetastorePrincipal = ClusterHelper.getPropertyValue( + srcCluster, SecurityUtil.HIVE_METASTORE_KERBEROS_PRINCIPAL); + if (StringUtils.isBlank(srcHiveMetastorePrincipal)) { + srcHiveMetastorePrincipal = NOT_APPLICABLE; } + additionalProperties.put(HiveMirroringExtensionProperties.SOURCE_NN_KERBEROS_PRINCIPAL.getName(), + srcClusterKerberosPrincipal); + additionalProperties.put( + HiveMirroringExtensionProperties.SOURCE_HIVE_METASTORE_KERBEROS_PRINCIPAL.getName(), + srcHiveMetastorePrincipal); // Properties for target cluster String targetClusterName = extensionProperties.getProperty(HiveMirroringExtensionProperties.TARGET_CLUSTER @@ -192,20 +199,23 @@ public class HiveMirroringExtension extends AbstractExtension { additionalProperties.put(HiveMirroringExtensionProperties.TARGET_NN.getName(), ClusterHelper.getStorageUrl(targetCluster)); - if (SecurityUtil.isSecurityEnabled()) { - String hive2Principal = extensionProperties.getProperty(HiveMirroringExtensionProperties - .TARGET_HIVE2_KERBEROS_PRINCIPAL.getName()); - if (StringUtils.isBlank(hive2Principal)) { - throw new FalconException("Hive server2 kerberos principal for cluster " + targetCluster.getName() - + "not passed for extension " + jobName); - } + // Check if target cluster is secure + String targetClusterKerberosPrincipal = ClusterHelper.getPropertyValue( + targetCluster, SecurityUtil.NN_PRINCIPAL); + if (StringUtils.isBlank(targetClusterKerberosPrincipal)) { + targetClusterKerberosPrincipal = NOT_APPLICABLE; + } - additionalProperties.put(HiveMirroringExtensionProperties.TARGET_NN_KERBEROS_PRINCIPAL.getName(), - ClusterHelper.getPropertyValue(targetCluster, SecurityUtil.NN_PRINCIPAL)); - additionalProperties.put( - HiveMirroringExtensionProperties.TARGET_HIVE_METASTORE_KERBEROS_PRINCIPAL.getName(), - ClusterHelper.getPropertyValue(targetCluster, SecurityUtil.HIVE_METASTORE_KERBEROS_PRINCIPAL)); + String targetHiveMetastorePrincipal = ClusterHelper.getPropertyValue(targetCluster, + SecurityUtil.HIVE_METASTORE_KERBEROS_PRINCIPAL); + if (StringUtils.isBlank(targetHiveMetastorePrincipal)) { + targetHiveMetastorePrincipal = NOT_APPLICABLE; } + additionalProperties.put(HiveMirroringExtensionProperties.TARGET_NN_KERBEROS_PRINCIPAL.getName(), + targetClusterKerberosPrincipal); + additionalProperties.put( + HiveMirroringExtensionProperties.TARGET_HIVE_METASTORE_KERBEROS_PRINCIPAL.getName(), + targetHiveMetastorePrincipal); // Misc properties // Add default properties if not passed
