Repository: ambari
Updated Branches:
  refs/heads/trunk e8e9781a8 -> 927d5c834


AMBARI-20122 - Stack advisor needs to recommend dependency for slaves and 
masters


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/927d5c83
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/927d5c83
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/927d5c83

Branch: refs/heads/trunk
Commit: 927d5c8340e5a22cd2fb8b8a2d14956b19810b93
Parents: e8e9781
Author: Tim Thorpe <ttho...@apache.org>
Authored: Mon Jun 19 09:00:20 2017 -0700
Committer: Tim Thorpe <ttho...@apache.org>
Committed: Mon Jun 19 09:00:20 2017 -0700

----------------------------------------------------------------------
 .../src/main/resources/stacks/stack_advisor.py  | 116 +++++++++++++-
 .../stacks/2.0.6/common/test_stack_advisor.py   | 153 +++++++++++++++++++
 2 files changed, 264 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/927d5c83/ambari-server/src/main/resources/stacks/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py 
b/ambari-server/src/main/resources/stacks/stack_advisor.py
index 4a81dc6..3a39a34 100644
--- a/ambari-server/src/main/resources/stacks/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/stack_advisor.py
@@ -756,15 +756,19 @@ class DefaultStackAdvisor(StackAdvisor):
       if hostName not in hostsComponentsMap:
         hostsComponentsMap[hostName] = []
 
+    #Sort the services so that the dependent services will be processed before 
those that depend on them.
+    sortedServices = self.getServicesSortedByDependencies(services)
     #extend hostsComponentsMap' with MASTER components
-    for service in services["services"]:
+    for service in sortedServices:
       masterComponents = [component for component in service["components"] if 
self.isMasterComponent(component)]
       serviceName = service["StackServices"]["service_name"]
       serviceAdvisor = self.getServiceAdvisor(serviceName)
       for component in masterComponents:
         componentName = component["StackServiceComponents"]["component_name"]
         advisor = serviceAdvisor if serviceAdvisor is not None else self
-        hostsForComponent = advisor.getHostsForMasterComponent(services, 
hosts, component, hostsList)
+        #Filter the hosts such that only hosts that meet the dependencies are 
included (if possible)
+        filteredHosts = self.getFilteredHostsBasedOnDependencies(services, 
component, hostsList, hostsComponentsMap)
+        hostsForComponent = advisor.getHostsForMasterComponent(services, 
hosts, component, filteredHosts)
 
         #extend 'hostsComponentsMap' with 'hostsForComponent'
         for hostName in hostsForComponent:
@@ -778,7 +782,7 @@ class DefaultStackAdvisor(StackAdvisor):
     utilizedHosts = [item for sublist in usedHostsListList for item in sublist]
     freeHosts = [hostName for hostName in hostsList if hostName not in 
utilizedHosts]
 
-    for service in services["services"]:
+    for service in sortedServices:
       slaveClientComponents = [component for component in service["components"]
                                if self.isSlaveComponent(component) or 
self.isClientComponent(component)]
       serviceName = service["StackServices"]["service_name"]
@@ -786,7 +790,10 @@ class DefaultStackAdvisor(StackAdvisor):
       for component in slaveClientComponents:
         componentName = component["StackServiceComponents"]["component_name"]
         advisor = serviceAdvisor if serviceAdvisor is not None else self
-        hostsForComponent = advisor.getHostsForSlaveComponent(services, hosts, 
component, hostsList, freeHosts)
+        #Filter the hosts and free hosts such that only hosts that meet the 
dependencies are included (if possible)
+        filteredHosts = self.getFilteredHostsBasedOnDependencies(services, 
component, hostsList, hostsComponentsMap)
+        filteredFreeHosts = self.filterList(freeHosts, filteredHosts)
+        hostsForComponent = advisor.getHostsForSlaveComponent(services, hosts, 
component, filteredHosts, filteredFreeHosts)
 
         #extend 'hostsComponentsMap' with 'hostsForComponent'
         for hostName in hostsForComponent:
@@ -796,7 +803,7 @@ class DefaultStackAdvisor(StackAdvisor):
             hostsComponentsMap[hostName].append( { "name": componentName } )
 
     #colocate custom services
-    for service in services["services"]:
+    for service in sortedServices:
       serviceName = service["StackServices"]["service_name"]
       serviceAdvisor = self.getServiceAdvisor(serviceName)
       if serviceAdvisor is not None:
@@ -866,6 +873,105 @@ class DefaultStackAdvisor(StackAdvisor):
 
     return hostsForComponent
 
+  def getServicesSortedByDependencies(self, services):
+    """
+    Sorts the services based on their dependencies.  This is limited to 
non-conditional host scope dependencies.
+    Services with no dependencies will go first.  Services with dependencies 
will go after the services they are dependent on.
+    If there are circular dependencies, the services will go in the order in 
which they were processed.
+    """
+    processedServices = []
+    sortedServices = []
+
+    for service in services["services"]:
+      self.sortServicesByDependencies(services, service, processedServices, 
sortedServices)
+
+    return sortedServices
+
+  def sortServicesByDependencies(self, services, service, processedServices, 
sortedServices):
+    """
+    Sorts the services based on their dependencies.  This is limited to 
non-conditional host scope dependencies.
+    Services with no dependencies will go first.  Services with dependencies 
will go after the services they are dependent on.
+    If there are circular dependencies, the services will go in the order in 
which they were processed.
+    """
+    if service is None or service in processedServices:
+      return
+
+    processedServices.append(service)
+
+    components = [] if "components" not in service else service["components"]
+    for component in components:
+      dependencies = [] if "dependencies" not in component else 
component['dependencies']
+      for dependency in dependencies:
+        # accounts only for dependencies that are not conditional
+        conditionsPresent =  "conditions" in dependency["Dependencies"] and 
dependency["Dependencies"]["conditions"]
+        scope = "cluster" if "scope" not in dependency["Dependencies"] else 
dependency["Dependencies"]["scope"]
+        if not conditionsPresent and scope == "host":
+          componentName = component["StackServiceComponents"]["component_name"]
+          requiredComponentName = dependency["Dependencies"]["component_name"]
+          requiredService = self.getServiceForComponentName(services, 
requiredComponentName)
+          self.sortServicesByDependencies(services, requiredService, 
processedServices, sortedServices)
+
+    sortedServices.append(service)
+
+  def getFilteredHostsBasedOnDependencies(self, services, component, 
hostsList, hostsComponentsMap):
+    """
+    Returns a list of hosts that only includes the ones which have all host 
scope dependencies already assigned to them.
+    If an empty list would be returned, instead the full list of hosts are 
returned.
+    In that case, we can't possibly return a valid recommended layout so we 
will at least return a fully filled layout.
+    """
+    removeHosts = []
+    dependencies = [] if "dependencies" not in component else 
component['dependencies']
+    for dependency in dependencies:
+      # accounts only for dependencies that are not conditional
+      conditionsPresent =  "conditions" in dependency["Dependencies"] and 
dependency["Dependencies"]["conditions"]
+      if not conditionsPresent:
+        componentName = component["StackServiceComponents"]["component_name"]
+        requiredComponentName = dependency["Dependencies"]["component_name"]
+        requiredComponent = self.getRequiredComponent(services, 
requiredComponentName)
+
+        # We only deal with "host" scope.
+        if (requiredComponent is not None) and 
(requiredComponent["component_category"] != "CLIENT"):
+          scope = "cluster" if "scope" not in dependency["Dependencies"] else 
dependency["Dependencies"]["scope"]
+          if scope == "host":
+            for host, hostComponents in hostsComponentsMap.iteritems():
+              isRequiredIncluded = False
+              for component in hostComponents:
+                currentComponentName = None if "name" not in component else 
component["name"]
+                if requiredComponentName == currentComponentName:
+                  isRequiredIncluded = True
+              if not isRequiredIncluded:
+                removeHosts.append(host)
+
+    filteredHostsList = []
+    for host in hostsList:
+      if host not in removeHosts:
+        filteredHostsList.append(host)
+    return filteredHostsList
+
+  def filterList(self, list, filter):
+    """
+    Returns the union of the two lists passed in (list and filter params).
+    """
+    filteredList = []
+    for item in list:
+      if item in filter:
+        filteredList.append(item)
+    return filteredList
+
+  def getServiceForComponentName(self, services, componentName):
+    """
+    Return service for component name
+
+    :type services dict
+    :type componentName str
+    """
+    for service in services["services"]:
+      for component in service["components"]:
+        if self.getComponentName(component) == componentName:
+          return service
+
+    return None
+
   def isComponentUsingCardinalityForLayout(self, componentName):
     return False
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/927d5c83/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py 
b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index 41c57f6..b6f1965 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -191,6 +191,159 @@ class TestHDP206StackAdvisor(TestCase):
     ]
     self.assertValidationResult(expectedItems, result)
 
+
+  def test_handleComponentDependencies(self):
+    services = {
+      "Versions":
+        {
+          "stack_name":"HDP",
+          "stack_version":"2.0.6"
+        },
+      "services" : [
+        {
+          "StackServices" : {
+            "service_name" : "HDFS",
+            "service_version" : "2.0.6",
+          },
+          "components": [
+            {
+              "StackServiceComponents": {
+                "stack_version": "2.0.6",
+                "stack_name": "HDP",
+                "component_category": "MASTER",
+                "is_client": False,
+                "is_master": True,
+                "service_name": "HDFS",
+                "cardinality": "1-2",
+                "hostnames": [],
+                "component_name": "NAMENODE",
+                "display_name": "NameNode"
+              },
+              "dependencies": [
+                {
+                  "Dependencies": {
+                    "stack_name": "HDP",
+                    "stack_version": "2.0.6",
+                    "scope": "cluster",
+                    "conditions": [
+                      {
+                        "configType": "hdfs-site",
+                        "property": "dfs.nameservices",
+                        "type": "PropertyExists",
+                      }
+                    ],
+                    "dependent_service_name": "HDFS",
+                    "dependent_component_name": "NAMENODE",
+                    "component_name": "ZOOKEEPER_SERVER"
+                  }
+                }
+              ]
+            }
+          ]
+        },
+        {
+          "StackServices" : {
+            "service_name" : "ZOOKEEPER",
+            "service_version" : "2.0.6",
+          },
+          "components": [
+            {
+              "StackServiceComponents": {
+                "stack_version": "2.0.6",
+                "stack_name": "HDP",
+                "component_category": "MASTER",
+                "is_client": False,
+                "is_master": True,
+                "service_name": "HDFS",
+                "cardinality": "1-2",
+                "hostnames": [],
+                "component_name": "ZOOKEEPER_SERVER",
+                "display_name": "ZooKeeper Server"
+              },
+              "dependencies": []
+            }
+          ]
+        }
+      ]
+    }
+
+    nameNodeDependencies = 
services["services"][0]["components"][0]["dependencies"][0]["Dependencies"]
+
+    # Tests for master component with dependencies
+
+    hosts = self.prepareHosts(["c6401.ambari.apache.org", 
"c6402.ambari.apache.org", "c6403.ambari.apache.org", 
"c6404.ambari.apache.org"])
+    
services["services"][1]["components"][0]["StackServiceComponents"]["hostnames"] 
= ["c6402.ambari.apache.org", "c6403.ambari.apache.org"]
+    recommendations = 
self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
+    # Assert that dependencies are ignored when there are conditions and 
cluster scope
+    
self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'],
 'NAMENODE')
+    
self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']),
 1)
+
+    nameNodeDependencies["scope"] = "host"
+    recommendations = 
self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
+    # Assert that dependencies are ignored when there are conditions (even for 
host scope)
+    
self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'],
 'NAMENODE')
+    
self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']),
 1)
+
+    nameNodeDependencies["scope"] = "cluster"
+    originalConditions = nameNodeDependencies["conditions"]
+    nameNodeDependencies["conditions"] = []
+    recommendations = 
self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
+    # Assert that dependencies are ignored when scope is cluster
+    
self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'],
 'NAMENODE')
+    
self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']),
 1)
+
+    nameNodeDependencies["scope"] = "host"
+    recommendations = 
self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
+    # Assert that dependencies are enforced for host scope without conditions
+    #self.assertEquals(recommendations, "")
+    
self.assertEquals(len(recommendations['blueprint']['host_groups'][0]['components']),
 2)
+
+    
services["services"][1]["components"][0]["StackServiceComponents"]["is_master"] 
= False
+    
services["services"][1]["components"][0]["StackServiceComponents"]["component_category"]
 = "CLIENT"
+    recommendations = 
self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
+    # Assert that dependencies are ignored when depending on client components
+    
self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'],
 'NAMENODE')
+    
self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']),
 1)
+
+    # Tests for slave component with dependencies
+    
services["services"][0]["components"][0]["StackServiceComponents"]["component_category"]
 = "SLAVE"
+    
services["services"][0]["components"][0]["StackServiceComponents"]["is_master"] 
= False
+    
services["services"][1]["components"][0]["StackServiceComponents"]["component_category"]
 = "MASTER"
+    
services["services"][1]["components"][0]["StackServiceComponents"]["is_master"] 
= True
+
+    nameNodeDependencies["scope"] = "cluster"
+    nameNodeDependencies["conditions"] = originalConditions
+    recommendations = 
self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
+    # Assert that dependencies are ignored when there are conditions and 
cluster scope
+    
self.assertEquals(recommendations['blueprint']['host_groups'][2]['components'][0]['name'],
 'NAMENODE')
+    
self.assertEquals(len(recommendations['blueprint']['host_groups'][2]['components']),
 1)
+    
self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'],
 'NAMENODE')
+    
self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']),
 1)
+
+    nameNodeDependencies["scope"] = "host"
+    recommendations = 
self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
+    # Assert that dependencies are ignored when there are conditions (even for 
host scope)
+    
self.assertEquals(recommendations['blueprint']['host_groups'][2]['components'][0]['name'],
 'NAMENODE')
+    
self.assertEquals(len(recommendations['blueprint']['host_groups'][2]['components']),
 1)
+    
self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'],
 'NAMENODE')
+    
self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']),
 1)
+
+    nameNodeDependencies["scope"] = "cluster"
+    nameNodeDependencies["conditions"] = []
+    recommendations = 
self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
+    # Assert that dependencies are ignored when scope is cluster
+    
self.assertEquals(recommendations['blueprint']['host_groups'][2]['components'][0]['name'],
 'NAMENODE')
+    
self.assertEquals(len(recommendations['blueprint']['host_groups'][2]['components']),
 1)
+    
self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'],
 'NAMENODE')
+    
self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']),
 1)
+
+    nameNodeDependencies["scope"] = "host"
+    recommendations = 
self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
+    # Assert that dependencies are enforced when host scope and no conditions
+    
self.assertEquals(recommendations['blueprint']['host_groups'][1]['components'][1]['name'],
 'NAMENODE')
+    
self.assertEquals(len(recommendations['blueprint']['host_groups'][1]['components']),
 2)
+
+
   def test_validateRequiredComponentsPresent(self):
     services = {
       "Versions":

Reply via email to