Repository: ambari Updated Branches: refs/heads/trunk 1be1d0495 -> b3f72e1e5
AMBARI-16752: Add pxf-hbase.jar to HBASE_CLASSPATH through service_advisor (mithmatt) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b3f72e1e Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b3f72e1e Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b3f72e1e Branch: refs/heads/trunk Commit: b3f72e1e5d89369e5f15538e90a977baf4f9be37 Parents: 1be1d04 Author: Matt <[email protected]> Authored: Thu May 19 15:04:06 2016 -0700 Committer: Matt <[email protected]> Committed: Thu May 19 15:04:06 2016 -0700 ---------------------------------------------------------------------- .../PXF/3.0.0/service_advisor.py | 33 +++++- .../stacks/2.3/PXF/test_service_advisor.py | 116 +++++++++++++++++++ 2 files changed, 144 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b3f72e1e/ambari-server/src/main/resources/common-services/PXF/3.0.0/service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PXF/3.0.0/service_advisor.py b/ambari-server/src/main/resources/common-services/PXF/3.0.0/service_advisor.py index e2510eb..9b34171 100644 --- a/ambari-server/src/main/resources/common-services/PXF/3.0.0/service_advisor.py +++ b/ambari-server/src/main/resources/common-services/PXF/3.0.0/service_advisor.py @@ -17,10 +17,7 @@ See the License for the specific language governing permissions and limitations under the License. """ import os -import fnmatch import imp -import socket -import sys import traceback SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -52,8 +49,6 @@ class PXF300ServiceAdvisor(service_advisor.ServiceAdvisor): def getComponentLayoutValidations(self, stackAdvisor, services, hosts): componentsListList = [service["components"] for service in services["services"]] componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist] - hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] - hostsCount = len(hostsList) pxfHosts = self.getHosts(componentsList, "PXF") expectedPxfHosts = set(self.getHosts(componentsList, "NAMENODE") + self.getHosts(componentsList, "DATANODE")) @@ -69,3 +64,31 @@ class PXF300ServiceAdvisor(service_advisor.ServiceAdvisor): items.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'PXF' } ) return items + + def getServiceConfigurationRecommendations(self, stackAdvisor, configurations, clusterData, services, hosts): + if "hbase-env" in services["configurations"]: + hbase_env = services["configurations"]["hbase-env"]["properties"] + if "content" in hbase_env: + content = hbase_env["content"] + PXF_PATH = "export HBASE_CLASSPATH=${HBASE_CLASSPATH}:/usr/lib/pxf/pxf-hbase.jar" + if "pxf-hbase.jar" not in content: + PXF_PATH = "#Add pxf-hbase.jar to HBASE_CLASSPATH\n" + PXF_PATH + content = "\n\n".join((content, PXF_PATH)) + putHbaseEnvProperty = self.putProperty(configurations, "hbase-env", services) + putHbaseEnvProperty("content", content) + + def validatePXFHBaseEnvConfigurations(self, stackAdvisor, properties, recommendedDefaults, configurations, services, hosts): + # Check if HBASE_CLASSPATH should has the location of pxf-hbase.jar + hbase_env = properties + validationItems = [] + if "content" in hbase_env and "pxf-hbase.jar" not in hbase_env["content"]: + message = "HBASE_CLASSPATH must contain the location of pxf-hbase.jar" + validationItems.append({"config-name": "content", "item": self.getWarnItem(message)}) + + return stackAdvisor.toConfigurationValidationProblems(validationItems, "hbase-env") + + def getConfigurationsValidationItems(self, stackAdvisor, configurations, recommendedDefaults, services, hosts): + siteName = "hbase-env" + method = self.validatePXFHBaseEnvConfigurations + items = self.validateConfigurationsForSite(stackAdvisor, configurations, recommendedDefaults, services, hosts, siteName, method) + return items \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/b3f72e1e/ambari-server/src/test/python/stacks/2.3/PXF/test_service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/PXF/test_service_advisor.py b/ambari-server/src/test/python/stacks/2.3/PXF/test_service_advisor.py new file mode 100644 index 0000000..c3a63cc --- /dev/null +++ b/ambari-server/src/test/python/stacks/2.3/PXF/test_service_advisor.py @@ -0,0 +1,116 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +from unittest import TestCase + + +class TestPXF300ServiceAdvisor(TestCase): + + def setUp(self): + import imp + self.testDirectory = os.path.dirname(os.path.abspath(__file__)) + stackAdvisorPath = os.path.join(self.testDirectory, '../../../../../main/resources/stacks/stack_advisor.py') + pxf300ServiceAdvisorPath = os.path.join(self.testDirectory, '../../../../../main/resources/common-services/PXF/3.0.0/service_advisor.py') + + with open(stackAdvisorPath, 'rb') as fp: + stack_advisor = imp.load_module('stack_advisor', fp, stackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE)) + with open(pxf300ServiceAdvisorPath, 'rb') as fp: + service_advisor = imp.load_module('stack_advisor_impl', fp, pxf300ServiceAdvisorPath, ('.py', 'rb', imp.PY_SOURCE)) + + stackAdvisorClass = getattr(stack_advisor, 'DefaultStackAdvisor') + self.stackAdvisor = stackAdvisorClass() + + serviceAdvisorClass = getattr(service_advisor, 'PXF300ServiceAdvisor') + self.serviceAdvisor = serviceAdvisorClass() + + self.PXF_PATH = "export HBASE_CLASSPATH=${HBASE_CLASSPATH}:/usr/lib/pxf/pxf-hbase.jar" + + def test_getServiceConfigurationRecommendations(self): + services = { + "configurations": { + "hbase-env": { + "properties": { + "content": "# Some hbase-env content text" + } + } + } + } + + ## Test is PXF_PATH is being added to hbase-env content + + # Case 1: Test pxf-hbase.jar classpath line was added to content + expected = "# Some hbase-env content text\n\n#Add pxf-hbase.jar to HBASE_CLASSPATH\n" + self.PXF_PATH + self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, services["configurations"], None, services, None) + self.assertEquals(services["configurations"]["hbase-env"]["properties"]["content"], expected) + + # Case 2: Test pxf-hbase.jar classpath line is not added again if content already has it + services["configurations"]["hbase-env"]["properties"]["content"] = self.PXF_PATH + expected = self.PXF_PATH + self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, services["configurations"], None, services, None) + self.assertEquals(services["configurations"]["hbase-env"]["properties"]["content"], expected) + + def test_getConfigurationsValidationItems(self): + services = { + "services": [ + { + "StackServices": { + "service_name": "PXF", + "service_version": "2.0", + "stack_name": "HDP", + "stack_version": "2.3" + } + }, + { + "StackServices": { + "service_name": "HBASE", + "service_version": "2.0", + "stack_name": "HDP", + "stack_version": "2.3" + } + } + ], + "configurations": { + "hbase-env": { + "properties": { + "content": "# Some hbase-env content text" + } + } + } + } + properties = services["configurations"] + + ## Test if PXF_PATH present in hbase-env content + + # Case 1: Generate warning item if PXF_PATH is not present in hbase-env + expected = [ + { + "config-type": "hbase-env", + "message": "HBASE_CLASSPATH must contain the location of pxf-hbase.jar", + "type": "configuration", + "config-name": "content", + "level": "WARN" + } + ] + items = self.serviceAdvisor.getConfigurationsValidationItems(self.stackAdvisor, properties, properties, services, None) + self.assertEquals(items, expected) + + # Case 2: No warning should be generated if PXF_PATH is present in hbase-env + properties = services["configurations"]["hbase-env"]["properties"]["content"] = self.PXF_PATH + items = self.serviceAdvisor.getConfigurationsValidationItems(self.stackAdvisor, properties, properties, services, None) + self.assertEquals(items, []) \ No newline at end of file
