AMBARI-22484. Stack advisor should disallow lzo enable without accepting license agreement. (Myroslav Papirkovskyy via swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9d55c0f5 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9d55c0f5 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9d55c0f5 Branch: refs/heads/branch-2.6 Commit: 9d55c0f5034c2765e4428d9b3ad56c455be54bfc Parents: 1fce565 Author: Siddharth Wagle <[email protected]> Authored: Mon Nov 20 10:58:47 2017 -0800 Committer: Siddharth Wagle <[email protected]> Committed: Mon Nov 20 10:58:47 2017 -0800 ---------------------------------------------------------------------- .../stackadvisor/StackAdvisorRequest.java | 11 ++++++++ .../commands/StackAdvisorCommand.java | 2 ++ .../ambari/server/controller/AmbariServer.java | 3 ++- .../internal/StackAdvisorResourceProvider.java | 9 +++++-- .../GPLLicenseNotAcceptedException.java | 28 ++++++++++++++++++++ .../stacks/HDP/2.0.6/services/stack_advisor.py | 23 +++++++++++++++- .../ValidationResourceProviderTest.java | 4 ++- .../stacks/2.0.6/common/test_stack_advisor.py | 25 +++++++++++++++++ 8 files changed, 100 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/9d55c0f5/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java index 64180e5..bc1e079 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java @@ -48,6 +48,7 @@ public class StackAdvisorRequest { private List<ChangedConfigInfo> changedConfigurations = new LinkedList<ChangedConfigInfo>(); private Set<RecommendationResponse.ConfigGroup> configGroups; private Map<String, String> userContext = new HashMap<String, String>(); + private Boolean gplLicenseAccepted; public String getStackName() { return stackName; @@ -117,6 +118,10 @@ public class StackAdvisorRequest { this.configGroups = configGroups; } + public Boolean getGplLicenseAccepted() { + return gplLicenseAccepted; + } + private StackAdvisorRequest(String stackName, String stackVersion) { this.stackName = stackName; this.stackVersion = stackVersion; @@ -189,6 +194,12 @@ public class StackAdvisorRequest { return this; } + public StackAdvisorRequestBuilder withGPLLicenseAccepted( + Boolean gplLicenseAccepted) { + this.instance.gplLicenseAccepted = gplLicenseAccepted; + return this; + } + public StackAdvisorRequest build() { return this.instance; } http://git-wip-us.apache.org/repos/asf/ambari/blob/9d55c0f5/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java index 5440462..48924f8 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java @@ -94,6 +94,7 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend private static final String CONFIGURATIONS_PROPERTY = "configurations"; private static final String CHANGED_CONFIGURATIONS_PROPERTY = "changed-configurations"; private static final String USER_CONTEXT_PROPERTY = "user-context"; + private static final String GPL_LICENSE_ACCEPTED = "gpl-license-accepted"; private static final String AMBARI_SERVER_CONFIGURATIONS_PROPERTY = "ambari-server-properties"; private File recommendationsDir; @@ -205,6 +206,7 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend JsonNode userContext = mapper.valueToTree(request.getUserContext()); root.put(USER_CONTEXT_PROPERTY, userContext); + root.put(GPL_LICENSE_ACCEPTED, request.getGplLicenseAccepted()); } private void populateConfigGroups(ObjectNode root, http://git-wip-us.apache.org/repos/asf/ambari/blob/9d55c0f5/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java index b2a68d6..154d4e7 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java @@ -904,7 +904,8 @@ public class AmbariServer { PersistKeyValueService.init(injector.getInstance(PersistKeyValueImpl.class)); KeyService.init(injector.getInstance(PersistKeyValueImpl.class)); BootStrapResource.init(injector.getInstance(BootStrapImpl.class)); - StackAdvisorResourceProvider.init(injector.getInstance(StackAdvisorHelper.class)); + StackAdvisorResourceProvider.init(injector.getInstance(StackAdvisorHelper.class), + injector.getInstance(Configuration.class)); StageUtils.setGson(injector.getInstance(Gson.class)); StageUtils.setTopologyManager(injector.getInstance(TopologyManager.class)); StageUtils.setConfiguration(injector.getInstance(Configuration.class)); http://git-wip-us.apache.org/repos/asf/ambari/blob/9d55c0f5/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java index dd32c6b..c2993db 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java @@ -36,6 +36,7 @@ import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest; import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest.StackAdvisorRequestBuilder; import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest.StackAdvisorRequestType; import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse; +import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.spi.Request; import org.apache.ambari.server.controller.spi.Resource.Type; @@ -79,12 +80,14 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi private static final String CONFIG_GROUPS_HOSTS_PROPERTY = "hosts"; protected static StackAdvisorHelper saHelper; + protected static Configuration configuration; protected static final String USER_CONTEXT_OPERATION_PROPERTY = "user_context/operation"; protected static final String USER_CONTEXT_OPERATION_DETAILS_PROPERTY = "user_context/operation_details"; @Inject - public static void init(StackAdvisorHelper instance) { + public static void init(StackAdvisorHelper instance, Configuration serverConfig) { saHelper = instance; + configuration = serverConfig; } protected StackAdvisorResourceProvider(Set<String> propertyIds, Map<Type, String> keyPropertyIds, @@ -130,6 +133,7 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi hgHostsMap); Map<String, Map<String, Map<String, String>>> configurations = calculateConfigurations(request); Map<String, String> userContext = readUserContext(request); + Boolean gplLicenseAccepted = configuration.getGplLicenseAccepted(); List<ChangedConfigInfo> changedConfigurations = requestType == StackAdvisorRequestType.CONFIGURATION_DEPENDENCIES ? @@ -144,7 +148,8 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi withConfigurations(configurations). withConfigGroups(configGroups). withChangedConfigurations(changedConfigurations). - withUserContext(userContext).build(); + withUserContext(userContext). + withGPLLicenseAccepted(gplLicenseAccepted).build(); } catch (Exception e) { LOG.warn("Error occurred during preparation of stack advisor request", e); Response response = Response.status(Status.BAD_REQUEST) http://git-wip-us.apache.org/repos/asf/ambari/blob/9d55c0f5/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java new file mode 100644 index 0000000..b444d01 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.topology; + +/** + * Indicates an not permitted LZO usage. + */ +public class GPLLicenseNotAcceptedException extends Exception { + public GPLLicenseNotAcceptedException(String s) { + super(s); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/9d55c0f5/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index c36a41f..6d80f1a 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -516,6 +516,22 @@ class HDP206StackAdvisor(DefaultStackAdvisor): # recommendations for "hadoop.proxyuser.*.hosts", "hadoop.proxyuser.*.groups" properties in core-site self.recommendHadoopProxyUsers(configurations, services, hosts) + def getLZOSupportValidationItems(self, properties, services): + services_list = self.get_services_list(services) + + if "HDFS" in services_list: + lzo_allowed = services["gpl-license-accepted"] + property_name = "io.compression.codec.lzo.class" + if property_name in properties: + property_value = properties.get(property_name) + if not lzo_allowed and "com.hadoop.compression.lzo.LzoCodec" in property_value: + return [{"config-name": property_name, "item": self.getErrorItem( + "Your Ambari Server has not been configured to download LZO and install it. " + "LZO is GPL software and requires you to accept a license prior to use. " + "Please refer to this documentation to configure Ambari before proceeding.")}] + + return [] + def recommendHbaseConfigurations(self, configurations, clusterData, services, hosts): # recommendations for HBase env config @@ -1173,7 +1189,8 @@ class HDP206StackAdvisor(DefaultStackAdvisor): def getServiceConfigurationValidators(self): return { "HDFS": { "hdfs-site": self.validateHDFSConfigurations, - "hadoop-env": self.validateHDFSConfigurationsEnv}, + "hadoop-env": self.validateHDFSConfigurationsEnv, + "core-site": self.validateHDFSConfigurationsCoreSite}, "MAPREDUCE2": {"mapred-site": self.validateMapReduce2Configurations}, "YARN": {"yarn-site": self.validateYARNConfigurations, "yarn-env": self.validateYARNEnvConfigurations}, @@ -1783,6 +1800,10 @@ class HDP206StackAdvisor(DefaultStackAdvisor): {"config-name": 'namenode_opt_maxnewsize', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'namenode_opt_maxnewsize')}] return self.toConfigurationValidationProblems(validationItems, "hadoop-env") + def validateHDFSConfigurationsCoreSite(self, properties, recommendedDefaults, configurations, services, hosts): + return self.toConfigurationValidationProblems(self.getLZOSupportValidationItems(properties, services), + "core-site") + def validatorOneDataDirPerPartition(self, properties, propertyName, services, hosts, clusterEnv): if not propertyName in properties: return self.getErrorItem("Value should be set") http://git-wip-us.apache.org/repos/asf/ambari/blob/9d55c0f5/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ValidationResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ValidationResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ValidationResourceProviderTest.java index cdbf2d7..e245e7f 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ValidationResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ValidationResourceProviderTest.java @@ -21,6 +21,7 @@ package org.apache.ambari.server.controller.internal; import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorHelper; import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest; import org.apache.ambari.server.api.services.stackadvisor.validations.ValidationResponse; +import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.spi.Request; import org.apache.ambari.server.controller.spi.RequestStatus; @@ -49,13 +50,14 @@ public class ValidationResourceProviderTest { doReturn(stackAdvisorRequest).when(provider).prepareStackAdvisorRequest(request); StackAdvisorHelper saHelper = mock(StackAdvisorHelper.class); + Configuration configuration = mock(Configuration.class); ValidationResponse response = mock(ValidationResponse.class); Version version = mock(Version.class); doReturn(3).when(response).getId(); doReturn(version).when(response).getVersion(); doReturn(response).when(saHelper).validate(any(StackAdvisorRequest.class)); - ValidationResourceProvider.init(saHelper); + ValidationResourceProvider.init(saHelper, configuration); RequestStatus status = provider.createResources(request); http://git-wip-us.apache.org/repos/asf/ambari/blob/9d55c0f5/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py index 9e00704..e011847 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py @@ -2356,6 +2356,31 @@ class TestHDP206StackAdvisor(TestCase): res = self.stackAdvisor.validateHDFSConfigurationsEnv(properties, recommendedDefaults, configurations, '', '') self.assertEquals(res, res_expected) + def test_validateHDFSConfigurationsCoreSite(self): + + configurations = {} + services = {"gpl-license-accepted": True, "services": [{"StackServices": {"service_name": "HDFS"}}]} + + # 1) ok: gpl is allowed + properties = {'io.compression.codec.lzo.class': 'com.hadoop.compression.lzo.LzoCodec'} + res_expected = [] + + res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, {}, configurations, services, '') + self.assertEquals(res, res_expected) + + # 2) fail: gpl is not allowed + services["gpl-license-accepted"] = False + res_expected = [{'config-type': 'core-site', + 'message': 'Your Ambari Server has not been configured to download LZO and install it. ' + 'LZO is GPL software and requires you to accept a license prior to use. ' + 'Please refer to this documentation to configure Ambari before proceeding.', + 'type': 'configuration', + 'config-name': 'io.compression.codec.lzo.class', + 'level': 'ERROR'}] + + res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, {}, configurations, services, '') + self.assertEquals(res, res_expected) + def test_validateOneDataDirPerPartition(self): recommendedDefaults = { 'dfs.datanode.du.reserved': '1024'
