AMBARI-17798. RU/EU: Add falcon to oozie admin user for HDP 2.5 (Venkat Ranganathan via alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/18ba883b Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/18ba883b Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/18ba883b Branch: refs/heads/branch-embedded-views Commit: 18ba883b314f2b8259a648b4ab1fd36f4a5cfda0 Parents: aff87ed Author: Alejandro Fernandez <[email protected]> Authored: Thu Jul 21 13:44:47 2016 -0700 Committer: Alejandro Fernandez <[email protected]> Committed: Thu Jul 21 13:44:47 2016 -0700 ---------------------------------------------------------------------- .../upgrades/FixOozieAdminUsers.java | 95 ++++++++++++ .../HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml | 1 + .../stacks/HDP/2.3/upgrades/upgrade-2.5.xml | 2 + .../HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml | 4 +- .../stacks/HDP/2.4/upgrades/upgrade-2.5.xml | 2 + .../stacks/HDP/2.5/services/stack_advisor.py | 47 ++++-- .../upgrades/FixOozieAdminUsersTest.java | 154 +++++++++++++++++++ 7 files changed, 287 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/18ba883b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixOozieAdminUsers.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixOozieAdminUsers.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixOozieAdminUsers.java new file mode 100644 index 0000000..3a06476 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixOozieAdminUsers.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.serveraction.upgrades; + +import com.google.inject.Inject; +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.actionmanager.HostRoleStatus; +import org.apache.ambari.server.agent.CommandReport; +import org.apache.ambari.server.serveraction.AbstractServerAction; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.Config; +import org.apache.commons.lang.StringUtils; + +import java.util.Map; +import java.util.concurrent.ConcurrentMap; + +/** + * During stack upgrade, update lzo codec path in mapreduce.application.classpath and + * at tez.cluster.additional.classpath.prefix to look like + * /usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar + */ +public class FixOozieAdminUsers extends AbstractServerAction { + private static final String TARGET_OOZIE_CONFIG_TYPE = "oozie-env"; + private static final String OOZIE_ADMIN_USERS_PROP = "oozie_admin_users"; + private static final String FALCON_CONFIG_TYPE = "falcon-env"; + private static final String FALCON_USER_PROP = "falcon_user"; + + + @Inject + private Clusters clusters; + + @Override + public CommandReport execute(ConcurrentMap<String, Object> requestSharedDataContext) + throws AmbariException, InterruptedException { + String clusterName = getExecutionCommand().getClusterName(); + Cluster cluster = clusters.getCluster(clusterName); + Config oozieConfig = cluster.getDesiredConfigByType(TARGET_OOZIE_CONFIG_TYPE); + Config falconConfig = cluster.getDesiredConfigByType(FALCON_CONFIG_TYPE); + + if (falconConfig == null) { + return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", + "Falcon configuration unavailable", ""); + + } + if (oozieConfig == null) { + return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", + "Oozie configuration unavailable", ""); + } + + Map<String, String> oozieProperties = oozieConfig.getProperties(); + String currentOozieAdmins = oozieProperties.get(OOZIE_ADMIN_USERS_PROP); + if (currentOozieAdmins.isEmpty()) { + currentOozieAdmins = ""; + } + Map<String, String> falconProperties = falconConfig.getProperties(); + String falconUser = falconProperties.get(FALCON_USER_PROP); + if (StringUtils.isBlank(falconUser)) { + return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", + "Falcon user not set", ""); + } + + if (currentOozieAdmins.indexOf(falconUser) >= 0) { + return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", + "Falcon user already member of Oozie admins", ""); + } + + String newOozieAdminUsers = currentOozieAdmins + "," + falconUser; + + oozieProperties.put(OOZIE_ADMIN_USERS_PROP, newOozieAdminUsers); + + oozieConfig.setProperties(oozieProperties); + oozieConfig.persist(false); + + return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", + String.format("Set oozie admin users to %s", newOozieAdminUsers), ""); + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/18ba883b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml index 7be5a19..a61b599 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml @@ -147,6 +147,7 @@ <task xsi:type="manual"> <message>Before continuing, please backup the Oozie Server database referenced by the Oozie server located on {{hosts.all}}.</message> </task> + <task xsi:type="server_action" summary="Fix oozie admin users" class="org.apache.ambari.server.serveraction.upgrades.FixOozieAdminUsers"/> </execute-stage> <execute-stage service="HIVE" component="HIVE_METASTORE" title="Backup Hive Metastore"> http://git-wip-us.apache.org/repos/asf/ambari/blob/18ba883b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml index 7f277fb..4d5fd07 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml @@ -883,6 +883,8 @@ <script>scripts/oozie_server_upgrade.py</script> <function>upgrade_oozie_database_and_sharelib</function> </task> + <task xsi:type="server_action" summary="Fixing Oozie admin users" + class="org.apache.ambari.server.serveraction.upgrades.FixOozieAdminUsers"/> </pre-upgrade> <pre-downgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/18ba883b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml index 2daa9b8..065a933 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml @@ -287,9 +287,11 @@ <!--OOZIE--> <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Apply config changes for Oozie Server"> <task xsi:type="server_action" summary="Adjusting Oozie properties" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation"/> + <task xsi:type="server_action" summary="Fix oozie admin users" class="org.apache.ambari.server.serveraction.upgrades.FixOozieAdminUsers"/> </execute-stage> - <!--FALCON--> + + <!--FALCON--> <execute-stage service="FALCON" component="FALCON_SERVER" title="Apply config changes for Falcon"> <task xsi:type="configure" id="hdp_2_5_0_0_falcon_server_adjust_services_property"/> </execute-stage> http://git-wip-us.apache.org/repos/asf/ambari/blob/18ba883b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml index 43c6265..d152f34 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml @@ -862,6 +862,8 @@ <script>scripts/oozie_server_upgrade.py</script> <function>upgrade_oozie_database_and_sharelib</function> </task> + <task xsi:type="server_action" summary="Fixing Oozie admin users" + class="org.apache.ambari.server.serveraction.upgrades.FixOozieAdminUsers"/> </pre-upgrade> <pre-downgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/18ba883b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index a7eed00..be2a011 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -36,36 +36,49 @@ class HDP25StackAdvisor(HDP24StackAdvisor): def recommendOozieConfigurations(self, configurations, clusterData, services, hosts): super(HDP25StackAdvisor,self).recommendOozieConfigurations(configurations, clusterData, services, hosts) putOozieEnvProperty = self.putProperty(configurations, "oozie-env", services) - if not "FALCON_SERVER" in clusterData["components"] : - print "Falcon is not part of the installation" - return if not "oozie-env" in services["configurations"] : - print "No oozie env in configurations" + Logger.info("No oozie configurations available") return - if "falcon-env" in services["configurations"] and "falcon_user" \ - in services["configurations"]["falcon-env"]["properties"] : - falconUser = services["configurations"]["falcon-env"]["properties"]["falcon_user"] - print "Falcon user from configuration " + falconUser - else : - falconUser = 'falcon' - print "Defaulting falcon user to " + falconUser + falconUser = 'falcon' + + if "falcon-env" in services["configurations"] : + if "falcon_user" in services["configurations"]["falcon-env"]["properties"] : + falconUser = services["configurations"]["falcon-env"]["properties"]["falcon_user"] + Logger.info("Falcon user from configuration: %s " % falconUser) + + Logger.info("Falcon user : %s" % falconUser) + + oozieUser = 'oozie' if "oozie_user" \ in services["configurations"]["oozie-env"]["properties"] : oozieUser = services["configurations"]["oozie-env"]["properties"]["oozie_user"] - print "Oozie user from configuration " + oozieUser + Logger.info("Oozie user from configuration %s" % oozieUser) + + Logger.info("Oozie user %s" % oozieUser) + + if "oozie_admin_users" \ + in services["configurations"]["oozie-env"]["properties"] : + currentAdminUsers = services["configurations"]["oozie-env"]["properties"]["oozie_admin_users"] + Logger.info("Oozie admin users from configuration %s" % currentAdminUsers) else : - oozieUser = 'oozie' - print "Defaulting oozie user to " + oozieUser + currentAdminUsers = "{0}, oozie-admin".format(oozieUser) + Logger.info("Setting default oozie admin users to %s" % currentAdminUsers) + + + if falconUser in currentAdminUsers : + Logger.info("Falcon user %s already member of oozie admin users " % falconUser) + return + + newAdminUsers = "{0},{1}".format(currentAdminUsers, falconUser) + + Logger.info("new oozie admin users : %s" % newAdminUsers) - newAdminUsers = "{0},oozie-admin,{1}".format(oozieUser, falconUser) - print "Setting new oozie admin user to " + newAdminUsers services["forced-configurations"].append({"type" : "oozie-env", "name" : "oozie_admin_users"}) putOozieEnvProperty("oozie_admin_users", newAdminUsers) - def createComponentLayoutRecommendations(self, services, hosts): parentComponentLayoutRecommendations = super(HDP25StackAdvisor, self).createComponentLayoutRecommendations( services, hosts) http://git-wip-us.apache.org/repos/asf/ambari/blob/18ba883b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/FixOozieAdminUsersTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/FixOozieAdminUsersTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/FixOozieAdminUsersTest.java new file mode 100644 index 0000000..314e955 --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/FixOozieAdminUsersTest.java @@ -0,0 +1,154 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.serveraction.upgrades; + +import com.google.inject.Injector; +import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper; +import org.apache.ambari.server.actionmanager.HostRoleCommand; +import org.apache.ambari.server.agent.CommandReport; +import org.apache.ambari.server.agent.ExecutionCommand; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.Config; +import org.apache.ambari.server.state.ConfigImpl; +import org.easymock.EasyMock; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.Map; + +import static org.easymock.EasyMock.*; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * Tests OozieConfigCalculation logic + */ +public class FixOozieAdminUsersTest { + private Injector injector; + private Clusters clusters; + private Cluster cluster; + private Field clustersField; + + @Before + public void setup() throws Exception { + injector = EasyMock.createMock(Injector.class); + clusters = EasyMock.createMock(Clusters.class); + cluster = EasyMock.createMock(Cluster.class); + + + Config falconEnvConfig = new ConfigImpl("falcon-env") { + Map<String, String> mockProperties = new HashMap<String, String>() {{ + put("falcon_user", "falcon"); + }}; + + @Override + public Map<String, String> getProperties() { + return mockProperties; + } + + @Override + public void setProperties(Map<String, String> properties) { + mockProperties.putAll(properties); + } + + @Override + public void persist(boolean newConfig) { + // no-op + } + }; + Config oozieEnvConfig = new ConfigImpl("oozie-env") { + Map<String, String> mockProperties = new HashMap<String, String>() {{ + put("oozie_admin_users", "oozie, oozie-admin"); + }}; + @Override + public Map<String, String> getProperties() { + return mockProperties; + } + + @Override + public void setProperties(Map<String, String> properties) { + mockProperties.putAll(properties); + } + + @Override + public void persist(boolean newConfig) { + // no-op + } + }; + expect(cluster.getDesiredConfigByType("falcon-env")).andReturn(falconEnvConfig).atLeastOnce(); + expect(cluster.getDesiredConfigByType("oozie-env")).andReturn(oozieEnvConfig).atLeastOnce(); + + expect(clusters.getCluster((String) anyObject())).andReturn(cluster).anyTimes(); + expect(injector.getInstance(Clusters.class)).andReturn(clusters).atLeastOnce(); + replay(injector, clusters); + + clustersField = FixOozieAdminUsers.class.getDeclaredField("clusters"); + clustersField.setAccessible(true); + + } + + /** + * Makes sure falcon user is added to oozie admin users + * @throws Exception + */ + @Test + public void testOozieAdminUserUpdated() throws Exception { + + Map<String, String> commandParams = new HashMap<String, String>(); + commandParams.put("clusterName", "c1"); + + ExecutionCommand executionCommand = new ExecutionCommand(); + executionCommand.setCommandParams(commandParams); + executionCommand.setClusterName("c1"); + + HostRoleCommand hrc = EasyMock.createMock(HostRoleCommand.class); + expect(hrc.getRequestId()).andReturn(1L).anyTimes(); + expect(hrc.getStageId()).andReturn(2L).anyTimes(); + expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)).anyTimes(); + replay(cluster, hrc); + + FixOozieAdminUsers action = new FixOozieAdminUsers(); + clustersField.set(action, clusters); + + action.setExecutionCommand(executionCommand); + action.setHostRoleCommand(hrc); + + CommandReport report = action.execute(null); + assertNotNull(report); + + + Cluster c = clusters.getCluster("c1"); + Config oozieConfig = c.getDesiredConfigByType("oozie-env"); + Config falconConfig = c.getDesiredConfigByType("falcon-env"); + + Map<String, String> oozieConfigMap = oozieConfig.getProperties(); + Map<String, String> falconConfigMap = falconConfig.getProperties(); + + assertTrue(oozieConfigMap.containsKey("oozie_admin_users")); + assertTrue(falconConfigMap.containsKey("falcon_user")); + String oozieAdminUsers = oozieConfigMap.get("oozie_admin_users"); + String falconUser = falconConfigMap.get("falcon_user"); + + assertTrue(oozieAdminUsers.indexOf(falconUser) != -1); + + } + +}
