AMBARI-22484. Stack advisor should disallow lzo enable without accepting license agreement. (mpapirkovskyy)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/41853a10 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/41853a10 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/41853a10 Branch: refs/heads/branch-3.0-perf Commit: 41853a1041a848b69384592c49f8f5b539a5c4f9 Parents: 7750398 Author: Myroslav Papirkovskyi <[email protected]> Authored: Wed Nov 29 16:48:11 2017 +0200 Committer: Myroslav Papirkovskyi <[email protected]> Committed: Wed Nov 29 18:30:08 2017 +0200 ---------------------------------------------------------------------- .../stackadvisor/StackAdvisorRequest.java | 11 +++++ .../commands/StackAdvisorCommand.java | 2 + .../ambari/server/controller/AmbariServer.java | 3 +- .../internal/StackAdvisorResourceProvider.java | 9 +++- .../GPLLicenseNotAcceptedException.java | 28 +++++++++++ .../stacks/HDP/2.0.6/services/stack_advisor.py | 17 +++++++ .../ValidationResourceProviderTest.java | 4 +- .../stacks/2.0.6/common/test_stack_advisor.py | 51 ++++++++++++++++++-- 8 files changed, 116 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/41853a10/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java index cd26c56..62b8d15 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java @@ -51,6 +51,7 @@ public class StackAdvisorRequest { private Set<RecommendationResponse.ConfigGroup> configGroups; private Map<String, String> userContext = new HashMap<>(); private Map<String, Object> ldapConfig = new HashMap<>(); + private Boolean gplLicenseAccepted; public String getStackName() { return stackName; @@ -122,6 +123,10 @@ public class StackAdvisorRequest { this.configGroups = configGroups; } + public Boolean getGplLicenseAccepted() { + return gplLicenseAccepted; + } + private StackAdvisorRequest(String stackName, String stackVersion) { this.stackName = stackName; this.stackVersion = stackVersion; @@ -194,6 +199,12 @@ public class StackAdvisorRequest { return this; } + public StackAdvisorRequestBuilder withGPLLicenseAccepted( + Boolean gplLicenseAccepted) { + this.instance.gplLicenseAccepted = gplLicenseAccepted; + return this; + } + public StackAdvisorRequestBuilder withLdapConfig(Map<String, Object> ldapConfig) { Preconditions.checkNotNull(ldapConfig); this.instance.ldapConfig = ldapConfig; http://git-wip-us.apache.org/repos/asf/ambari/blob/41853a10/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java index 1b89c4f..4ec15ef 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java @@ -107,6 +107,7 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend private static final String CONFIGURATIONS_PROPERTY = "configurations"; private static final String CHANGED_CONFIGURATIONS_PROPERTY = "changed-configurations"; private static final String USER_CONTEXT_PROPERTY = "user-context"; + private static final String GPL_LICENSE_ACCEPTED = "gpl-license-accepted"; private static final String AMBARI_SERVER_CONFIGURATIONS_PROPERTY = "ambari-server-properties"; private File recommendationsDir; @@ -269,6 +270,7 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend JsonNode userContext = mapper.valueToTree(request.getUserContext()); root.put(USER_CONTEXT_PROPERTY, userContext); + root.put(GPL_LICENSE_ACCEPTED, request.getGplLicenseAccepted()); } private void populateConfigGroups(ObjectNode root, http://git-wip-us.apache.org/repos/asf/ambari/blob/41853a10/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java index 6ceed4a..863313b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java @@ -908,7 +908,8 @@ public class AmbariServer { PersistKeyValueService.init(injector.getInstance(PersistKeyValueImpl.class)); KeyService.init(injector.getInstance(PersistKeyValueImpl.class)); BootStrapResource.init(injector.getInstance(BootStrapImpl.class)); - StackAdvisorResourceProvider.init(injector.getInstance(StackAdvisorHelper.class)); + StackAdvisorResourceProvider.init(injector.getInstance(StackAdvisorHelper.class), + injector.getInstance(Configuration.class)); StageUtils.setGson(injector.getInstance(Gson.class)); StageUtils.setTopologyManager(injector.getInstance(TopologyManager.class)); StageUtils.setConfiguration(injector.getInstance(Configuration.class)); http://git-wip-us.apache.org/repos/asf/ambari/blob/41853a10/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java index 1ea664c..67c177e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java @@ -36,6 +36,7 @@ import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest; import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest.StackAdvisorRequestBuilder; import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest.StackAdvisorRequestType; import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse; +import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.spi.Request; import org.apache.ambari.server.controller.spi.Resource.Type; @@ -82,12 +83,14 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi private static final String CONFIG_GROUPS_HOSTS_PROPERTY = "hosts"; protected static StackAdvisorHelper saHelper; + protected static Configuration configuration; protected static final String USER_CONTEXT_OPERATION_PROPERTY = "user_context/operation"; protected static final String USER_CONTEXT_OPERATION_DETAILS_PROPERTY = "user_context/operation_details"; @Inject - public static void init(StackAdvisorHelper instance) { + public static void init(StackAdvisorHelper instance, Configuration serverConfig) { saHelper = instance; + configuration = serverConfig; } protected StackAdvisorResourceProvider(Set<String> propertyIds, Map<Type, String> keyPropertyIds, @@ -133,6 +136,7 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi hgHostsMap); Map<String, Map<String, Map<String, String>>> configurations = calculateConfigurations(request); Map<String, String> userContext = readUserContext(request); + Boolean gplLicenseAccepted = configuration.getGplLicenseAccepted(); List<ChangedConfigInfo> changedConfigurations = requestType == StackAdvisorRequestType.CONFIGURATION_DEPENDENCIES ? @@ -147,7 +151,8 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi withConfigurations(configurations). withConfigGroups(configGroups). withChangedConfigurations(changedConfigurations). - withUserContext(userContext).build(); + withUserContext(userContext). + withGPLLicenseAccepted(gplLicenseAccepted).build(); } catch (Exception e) { LOG.warn("Error occurred during preparation of stack advisor request", e); Response response = Response.status(Status.BAD_REQUEST) http://git-wip-us.apache.org/repos/asf/ambari/blob/41853a10/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java new file mode 100644 index 0000000..b444d01 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.topology; + +/** + * Indicates an not permitted LZO usage. + */ +public class GPLLicenseNotAcceptedException extends Exception { + public GPLLicenseNotAcceptedException(String s) { + super(s); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/41853a10/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index a194332..bfa2f5a 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -351,6 +351,22 @@ class HDP206StackAdvisor(DefaultStackAdvisor): # recommendations for "hadoop.proxyuser.*.hosts", "hadoop.proxyuser.*.groups" properties in core-site self.recommendHadoopProxyUsers(configurations, services, hosts) + def getLZOSupportValidationItems(self, properties, services): + services_list = self.get_services_list(services) + + if "HDFS" in services_list: + lzo_allowed = services["gpl-license-accepted"] + property_name = "io.compression.codec.lzo.class" + if property_name in properties: + property_value = properties.get(property_name) + if not lzo_allowed and "com.hadoop.compression.lzo.LzoCodec" in property_value: + return [{"config-name": property_name, "item": self.getErrorItem( + "Your Ambari Server has not been configured to download LZO and install it. " + "LZO is GPL software and requires you to accept a license prior to use. " + "Please refer to this documentation to configure Ambari before proceeding.")}] + + return [] + def recommendHbaseConfigurations(self, configurations, clusterData, services, hosts): # recommendations for HBase env config @@ -726,6 +742,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): validationItems = [] validationItems.extend(self.getHadoopProxyUsersValidationItems(properties, services, hosts, configurations)) validationItems.extend(self.getAmbariProxyUsersForHDFSValidationItems(properties, services)) + validationItems.extend(self.getLZOSupportValidationItems(properties, services)) return self.toConfigurationValidationProblems(validationItems, "core-site") def validatorOneDataDirPerPartition(self, properties, propertyName, services, hosts, clusterEnv): http://git-wip-us.apache.org/repos/asf/ambari/blob/41853a10/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ValidationResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ValidationResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ValidationResourceProviderTest.java index c639d1f..42d4770 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ValidationResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ValidationResourceProviderTest.java @@ -33,6 +33,7 @@ import java.util.Set; import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorHelper; import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest; import org.apache.ambari.server.api.services.stackadvisor.validations.ValidationResponse; +import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.spi.Request; import org.apache.ambari.server.controller.spi.RequestStatus; @@ -53,13 +54,14 @@ public class ValidationResourceProviderTest { doReturn(stackAdvisorRequest).when(provider).prepareStackAdvisorRequest(request); StackAdvisorHelper saHelper = mock(StackAdvisorHelper.class); + Configuration configuration = mock(Configuration.class); ValidationResponse response = mock(ValidationResponse.class); Version version = mock(Version.class); doReturn(3).when(response).getId(); doReturn(version).when(response).getVersion(); doReturn(response).when(saHelper).validate(any(StackAdvisorRequest.class)); - ValidationResourceProvider.init(saHelper); + ValidationResourceProvider.init(saHelper, configuration); RequestStatus status = provider.createResources(request); http://git-wip-us.apache.org/repos/asf/ambari/blob/41853a10/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py index 38d6ecd..6c774af 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py @@ -2521,7 +2521,8 @@ class TestHDP206StackAdvisor(TestCase): 'hadoop.proxyuser.hdfs-user.hosts': '*', 'hadoop.proxyuser.hdfs-user.groups': '*', 'hadoop.proxyuser.yarn-user.hosts': 'host1,host2', - 'hadoop.proxyuser.yarn-user.groups': '*'} + 'hadoop.proxyuser.yarn-user.groups': '*', + 'io.compression.codec.lzo.class': 'com.hadoop.compression.lzo.LzoCodec'} services = { 'services': [ { 'StackServices': {'service_name': 'HDFS'}}, @@ -2537,7 +2538,8 @@ class TestHDP206StackAdvisor(TestCase): } ], 'ambari-server-properties': {'ambari-server.user': 'ambari-user'}, - 'configurations': configurations + 'configurations': configurations, + "gpl-license-accepted": True } hosts = { 'items' : [ @@ -2551,7 +2553,21 @@ class TestHDP206StackAdvisor(TestCase): res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, recommendedDefaults, configurations, services, hosts) self.assertEquals(res, res_expected) - # 2) fail: test filter function: two RESOURCE_MANAGERs, hadoop.proxyuser.yarn-user.hosts is expected to be set + # 2) fail: gpl is not allowed + services["gpl-license-accepted"] = False + res_expected = [{'config-type': 'core-site', + 'message': 'Your Ambari Server has not been configured to download LZO and install it. ' + 'LZO is GPL software and requires you to accept a license prior to use. ' + 'Please refer to this documentation to configure Ambari before proceeding.', + 'type': 'configuration', + 'config-name': 'io.compression.codec.lzo.class', + 'level': 'ERROR'}] + + res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, {}, configurations, services, hosts) + self.assertEquals(res, res_expected) + services["gpl-license-accepted"] = True + + # 3) fail: test filter function: two RESOURCE_MANAGERs, hadoop.proxyuser.yarn-user.hosts is expected to be set del properties['hadoop.proxyuser.yarn-user.hosts'] res_expected = [{'config-name': 'hadoop.proxyuser.yarn-user.hosts', 'config-type': 'core-site', @@ -2561,13 +2577,13 @@ class TestHDP206StackAdvisor(TestCase): res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, recommendedDefaults, configurations, services, hosts) self.assertEquals(res, res_expected) - # 3) ok: test filter function: only one RESOURCE_MANAGER + # 4) ok: test filter function: only one RESOURCE_MANAGER services['services'][1]['components'][0]['StackServiceComponents']['hostnames'] = ["host1"] res_expected = [] res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, recommendedDefaults, configurations, services, hosts) self.assertEquals(res, res_expected) - # 4) fail: some proxyusers are empty or absent: + # 5) fail: some proxyusers are empty or absent: del properties['hadoop.proxyuser.ambari-user.hosts'] properties['hadoop.proxyuser.hdfs-user.groups'] = '' res_expected = [{'config-name': 'hadoop.proxyuser.hdfs-user.groups', @@ -2627,6 +2643,31 @@ class TestHDP206StackAdvisor(TestCase): res = self.stackAdvisor.getHadoopProxyUsers(services, hosts, configurations) self.assertEquals(res, res_expected) + # def test_validateHDFSConfigurationsCoreSite(self): + # + # configurations = {} + # services = {"gpl-license-accepted": True, "services": [{"StackServices": {"service_name": "HDFS"}}], 'ambari-server-properties': {'ambari-server.user': 'ambari-user'}} + # + # # 1) ok: gpl is allowed + # properties = {'io.compression.codec.lzo.class': 'com.hadoop.compression.lzo.LzoCodec'} + # res_expected = [] + # + # res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, {}, configurations, services, '') + # self.assertEquals(res, res_expected) + # + # # 2) fail: gpl is not allowed + # services["gpl-license-accepted"] = False + # res_expected = [{'config-type': 'core-site', + # 'message': 'Your Ambari Server has not been configured to download LZO and install it. ' + # 'LZO is GPL software and requires you to accept a license prior to use. ' + # 'Please refer to this documentation to configure Ambari before proceeding.', + # 'type': 'configuration', + # 'config-name': 'io.compression.codec.lzo.class', + # 'level': 'ERROR'}] + # + # res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, {}, configurations, services, '') + # self.assertEquals(res, res_expected) + def test_validateOneDataDirPerPartition(self): recommendedDefaults = { 'dfs.datanode.du.reserved': '1024'
