Repository: ambari Updated Branches: refs/heads/trunk cdd97b08c -> 710a8fea4
AMBARI-15697. Allow skipping creation of repo files (ncole) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/710a8fea Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/710a8fea Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/710a8fea Branch: refs/heads/trunk Commit: 710a8fea44d56e71c4e21989021d9332b726e089 Parents: cdd97b0 Author: Nate Cole <[email protected]> Authored: Mon Apr 4 16:54:03 2016 -0400 Committer: Nate Cole <[email protected]> Committed: Tue Apr 5 07:51:08 2016 -0400 ---------------------------------------------------------------------- .../AmbariCustomCommandExecutionHelper.java | 13 +- .../AmbariManagementControllerImpl.java | 1 + .../controller/OperatingSystemResponse.java | 15 ++ .../ClusterStackVersionResourceProvider.java | 8 +- .../OperatingSystemResourceProvider.java | 7 +- .../orm/entities/OperatingSystemEntity.java | 13 ++ .../stack/upgrade/RepositoryVersionHelper.java | 13 +- .../custom_actions/scripts/install_packages.py | 4 + .../scripts/repo_initialization.py | 9 +- ...ClusterStackVersionResourceProviderTest.java | 218 +++++++++++++++++++ .../RepositoryVersionResourceProviderTest.java | 70 ++++++ .../upgrades/UpgradeActionTest.java | 17 +- .../custom_actions/TestInstallPackages.py | 51 +++++ .../hooks/before-INSTALL/test_before_install.py | 18 ++ 14 files changed, 446 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/710a8fea/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java index f3197cb..5e51f5f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java @@ -1034,7 +1034,7 @@ public class AmbariCustomCommandExecutionHelper { } if (null != gsonList) { - updateBaseUrls(cluster, JsonArray.class.cast(gsonList)); + gsonList = updateBaseUrls(cluster, JsonArray.class.cast(gsonList)); return gsonList.toString(); } else { return ""; @@ -1047,7 +1047,7 @@ public class AmbariCustomCommandExecutionHelper { * @param cluster the cluster to load the current version * @param jsonArray the array containing stack repo data */ - private void updateBaseUrls(Cluster cluster, JsonArray jsonArray) throws AmbariException { + private JsonArray updateBaseUrls(Cluster cluster, JsonArray jsonArray) throws AmbariException { ClusterVersionEntity cve = cluster.getCurrentClusterVersion(); if (null == cve) { @@ -1066,11 +1066,13 @@ public class AmbariCustomCommandExecutionHelper { if (null == cve || null == cve.getRepositoryVersion()) { LOG.info("Cluster {} has no specific Repository Versions. Using stack-defined values", cluster.getClusterName()); - return; + return jsonArray; } RepositoryVersionEntity rve = cve.getRepositoryVersion(); + JsonArray result = new JsonArray(); + for (JsonElement e : jsonArray) { JsonObject obj = e.getAsJsonObject(); @@ -1084,7 +1086,7 @@ public class AmbariCustomCommandExecutionHelper { } for (OperatingSystemEntity ose : rve.getOperatingSystems()) { - if (ose.getOsType().equals(osType)) { + if (ose.getOsType().equals(osType) && ose.isAmbariManagedRepos()) { for (RepositoryEntity re : ose.getRepositories()) { if (re.getName().equals(repoName) && re.getRepositoryId().equals(repoId) && @@ -1092,9 +1094,12 @@ public class AmbariCustomCommandExecutionHelper { obj.addProperty("baseUrl", re.getBaseUrl()); } } + result.add(e); } } } + + return result; } http://git-wip-us.apache.org/repos/asf/ambari/blob/710a8fea/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index a0b98f7..1793ef2 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -4248,6 +4248,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle } response.setStackName(repositoryVersion.getStackName()); response.setStackVersion(repositoryVersion.getStackVersion()); + response.setAmbariManagedRepos(operatingSystem.isAmbariManagedRepos()); responses.add(response); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/710a8fea/ambari-server/src/main/java/org/apache/ambari/server/controller/OperatingSystemResponse.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/OperatingSystemResponse.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/OperatingSystemResponse.java index 06b4148..0f0d79c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/OperatingSystemResponse.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/OperatingSystemResponse.java @@ -25,6 +25,7 @@ public class OperatingSystemResponse { private String osType; private Long repositoryVersionId; private String versionDefinitionId; + private boolean ambariManagedRepos = true; public OperatingSystemResponse(String osType) { setOsType(osType); @@ -75,4 +76,18 @@ public class OperatingSystemResponse { public String getVersionDefinitionId() { return versionDefinitionId; } + + /** + * @param managed {@code true} if ambari is managing the repositories for the OS + */ + public void setAmbariManagedRepos(boolean managed) { + ambariManagedRepos = managed; + } + + /** + * @return {@code true} if ambari is managing the repositories for the OS + */ + public boolean isAmbariManagedRepos() { + return ambariManagedRepos; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/710a8fea/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java index bb50820..a85f939 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java @@ -356,7 +356,13 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou List<OperatingSystemEntity> operatingSystems = repoVersionEnt.getOperatingSystems(); Map<String, List<RepositoryEntity>> perOsRepos = new HashMap<String, List<RepositoryEntity>>(); for (OperatingSystemEntity operatingSystem : operatingSystems) { - perOsRepos.put(operatingSystem.getOsType(), operatingSystem.getRepositories()); + + if (operatingSystem.isAmbariManagedRepos()) { + perOsRepos.put(operatingSystem.getOsType(), operatingSystem.getRepositories()); + } else { + perOsRepos.put(operatingSystem.getOsType(), Collections.<RepositoryEntity>emptyList()); + } + } RequestStageContainer req = createRequest(); http://git-wip-us.apache.org/repos/asf/ambari/blob/710a8fea/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/OperatingSystemResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/OperatingSystemResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/OperatingSystemResourceProvider.java index fd4a91f..fef94be 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/OperatingSystemResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/OperatingSystemResourceProvider.java @@ -47,6 +47,7 @@ public class OperatingSystemResourceProvider extends ReadOnlyResourceProvider { public static final String OPERATING_SYSTEM_OS_TYPE_PROPERTY_ID = PropertyHelper.getPropertyId("OperatingSystems", "os_type"); public static final String OPERATING_SYSTEM_REPOSITORY_VERSION_ID_PROPERTY_ID = PropertyHelper.getPropertyId("OperatingSystems", "repository_version_id"); public static final String OPERATING_SYSTEM_VERSION_DEFINITION_ID_PROPERTY_ID = PropertyHelper.getPropertyId("OperatingSystems", "version_definition_id"); + public static final String OPERATING_SYSTEM_AMBARI_MANAGED_REPOS = "OperatingSystems/ambari_managed_repositories"; private static Set<String> pkPropertyIds = Sets.newHashSet( OPERATING_SYSTEM_OS_TYPE_PROPERTY_ID, @@ -58,7 +59,8 @@ public class OperatingSystemResourceProvider extends ReadOnlyResourceProvider { OPERATING_SYSTEM_STACK_NAME_PROPERTY_ID, OPERATING_SYSTEM_STACK_VERSION_PROPERTY_ID, OPERATING_SYSTEM_REPOSITORY_VERSION_ID_PROPERTY_ID, - OPERATING_SYSTEM_VERSION_DEFINITION_ID_PROPERTY_ID); + OPERATING_SYSTEM_VERSION_DEFINITION_ID_PROPERTY_ID, + OPERATING_SYSTEM_AMBARI_MANAGED_REPOS); public static Map<Type, String> keyPropertyIds = new HashMap<Type, String>() { { @@ -113,6 +115,9 @@ public class OperatingSystemResourceProvider extends ReadOnlyResourceProvider { setResourceProperty(resource, OPERATING_SYSTEM_OS_TYPE_PROPERTY_ID, response.getOsType(), requestedIds); + setResourceProperty(resource, OPERATING_SYSTEM_AMBARI_MANAGED_REPOS, response.isAmbariManagedRepos(), + requestedIds); + if (response.getRepositoryVersionId() != null) { setResourceProperty(resource, OPERATING_SYSTEM_REPOSITORY_VERSION_ID_PROPERTY_ID, response.getRepositoryVersionId(), requestedIds); http://git-wip-us.apache.org/repos/asf/ambari/blob/710a8fea/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/OperatingSystemEntity.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/OperatingSystemEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/OperatingSystemEntity.java index 3c881a1..f615da6 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/OperatingSystemEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/OperatingSystemEntity.java @@ -27,6 +27,7 @@ public class OperatingSystemEntity { private String osType; private List<RepositoryEntity> repositories = new ArrayList<RepositoryEntity>(); + private boolean ambariManagedRepos = true; public String getOsType() { return osType; @@ -56,10 +57,22 @@ public class OperatingSystemEntity { return true; } + public void setAmbariManagedRepos(boolean managed) { + ambariManagedRepos = managed; + } + + /** + * @return + */ + public boolean isAmbariManagedRepos() { + return ambariManagedRepos; + } + @Override public int hashCode() { int result = osType != null ? osType.hashCode() : 0; return result; } + } http://git-wip-us.apache.org/repos/asf/ambari/blob/710a8fea/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java index 6a36522..50d6028 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java @@ -89,9 +89,18 @@ public class RepositoryVersionHelper { final List<OperatingSystemEntity> operatingSystems = new ArrayList<OperatingSystemEntity>(); final JsonArray rootJson = new JsonParser().parse(repositoriesJson).getAsJsonArray(); for (JsonElement operatingSystemJson: rootJson) { + JsonObject osObj = operatingSystemJson.getAsJsonObject(); + final OperatingSystemEntity operatingSystemEntity = new OperatingSystemEntity(); - operatingSystemEntity.setOsType(operatingSystemJson.getAsJsonObject().get(OperatingSystemResourceProvider.OPERATING_SYSTEM_OS_TYPE_PROPERTY_ID).getAsString()); - for (JsonElement repositoryJson: operatingSystemJson.getAsJsonObject().get(RepositoryVersionResourceProvider.SUBRESOURCE_REPOSITORIES_PROPERTY_ID).getAsJsonArray()) { + + operatingSystemEntity.setOsType(osObj.get(OperatingSystemResourceProvider.OPERATING_SYSTEM_OS_TYPE_PROPERTY_ID).getAsString()); + + if (osObj.has(OperatingSystemResourceProvider.OPERATING_SYSTEM_AMBARI_MANAGED_REPOS)) { + operatingSystemEntity.setAmbariManagedRepos(osObj.get( + OperatingSystemResourceProvider.OPERATING_SYSTEM_AMBARI_MANAGED_REPOS).getAsBoolean()); + } + + for (JsonElement repositoryJson: osObj.get(RepositoryVersionResourceProvider.SUBRESOURCE_REPOSITORIES_PROPERTY_ID).getAsJsonArray()) { final RepositoryEntity repositoryEntity = new RepositoryEntity(); repositoryEntity.setBaseUrl(repositoryJson.getAsJsonObject().get(RepositoryResourceProvider.REPOSITORY_BASE_URL_PROPERTY_ID).getAsString()); repositoryEntity.setName(repositoryJson.getAsJsonObject().get(RepositoryResourceProvider.REPOSITORY_REPO_NAME_PROPERTY_ID).getAsString()); http://git-wip-us.apache.org/repos/asf/ambari/blob/710a8fea/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py index ba5ce62..709781e 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py @@ -117,6 +117,10 @@ class InstallPackages(Script): self.current_repo_files.add('base') Logger.info("Will install packages for repository version {0}".format(self.repository_version)) + + if 0 == len(base_urls): + Logger.info("Repository list is empty. Ambari may not be managing the repositories for {0}.".format(self.repository_version)) + try: append_to_file = False for url_info in base_urls: http://git-wip-us.apache.org/repos/asf/ambari/blob/710a8fea/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/repo_initialization.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/repo_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/repo_initialization.py index 05751fa..a35dce7 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/repo_initialization.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/repo_initialization.py @@ -16,8 +16,10 @@ See the License for the specific language governing permissions and limitations under the License. """ -from resource_management import * + from ambari_commons.os_check import OSCheck +from resource_management.libraries.resources.repository import Repository +from resource_management.core.logger import Logger import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set. # components_lits = repoName + postfix @@ -33,6 +35,11 @@ def _alter_repo(action, repo_string, repo_template): if not isinstance(repo_dicts, list): repo_dicts = [repo_dicts] + if 0 == len(repo_dicts): + Logger.info("Repository list is empty. Ambari may not be managing the repositories.") + else: + Logger.info("Initializing {0} repositories".format(str(len(repo_dicts)))) + for repo in repo_dicts: if not 'baseUrl' in repo: repo['baseUrl'] = None http://git-wip-us.apache.org/repos/asf/ambari/blob/710a8fea/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java index 4da8896..3c9a91d 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java @@ -108,6 +108,9 @@ import org.junit.Before; import org.junit.Ignore; import org.junit.Test; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import com.google.inject.AbstractModule; import com.google.inject.Guice; import com.google.inject.Injector; @@ -787,6 +790,221 @@ public class ClusterStackVersionResourceProviderTest { Assert.assertTrue(executionCommand.getRoleParams().containsKey(KeyNames.PACKAGE_VERSION)); } + @Test + public void testCreateResourcesWithNonManagedOS() throws Exception { + JsonArray json = new JsonParser().parse(OS_JSON).getAsJsonArray(); + + JsonObject jsonObj = json.get(0).getAsJsonObject(); + jsonObj.addProperty(OperatingSystemResourceProvider.OPERATING_SYSTEM_AMBARI_MANAGED_REPOS, false); + + String os_json = json.toString(); + + Resource.Type type = Resource.Type.ClusterStackVersion; + + AmbariManagementController managementController = createMock(AmbariManagementController.class); + Clusters clusters = createNiceMock(Clusters.class); + Cluster cluster = createNiceMock(Cluster.class); + StackId stackId = new StackId("HDP", "2.0.1"); + + File f = new File("src/test/resources/hbase_version_test.xml"); + + RepositoryVersionEntity repoVersion = new RepositoryVersionEntity(); + repoVersion.setId(1l); + repoVersion.setOperatingSystems(os_json); + repoVersion.setVersionXml(IOUtils.toString(new FileInputStream(f))); + repoVersion.setVersionXsd("version_definition.xsd"); + repoVersion.setType(RepositoryType.STANDARD); + + ambariMetaInfo.getComponent("HDP", "2.1.1", "HBASE", "HBASE_MASTER").setVersionAdvertised(true); + + Map<String, Host> hostsForCluster = new HashMap<String, Host>(); + int hostCount = 10; + for (int i = 0; i < hostCount; i++) { + String hostname = "host" + i; + Host host = createNiceMock(hostname, Host.class); + expect(host.getHostName()).andReturn(hostname).anyTimes(); + expect(host.getOsFamily()).andReturn("redhat6").anyTimes(); + expect(host.getMaintenanceState(EasyMock.anyLong())).andReturn( + MaintenanceState.OFF).anyTimes(); + expect(host.getAllHostVersions()).andReturn( + Collections.<HostVersionEntity>emptyList()).anyTimes(); + + replay(host); + hostsForCluster.put(hostname, host); + } + + Service hdfsService = createNiceMock(Service.class); + Service hbaseService = createNiceMock(Service.class); + expect(hdfsService.getName()).andReturn("HDFS").anyTimes(); + expect(hbaseService.getName()).andReturn("HBASE").anyTimes(); + + expect(hdfsService.getServiceComponents()).andReturn(new HashMap<String, ServiceComponent>()); + expect(hbaseService.getServiceComponents()).andReturn(new HashMap<String, ServiceComponent>()); + + Map<String, Service> serviceMap = new HashMap<>(); + serviceMap.put("HDFS", hdfsService); + serviceMap.put("HBASE", hbaseService); + + final ServiceComponentHost schDatanode = createMock(ServiceComponentHost.class); + expect(schDatanode.getServiceName()).andReturn("HDFS").anyTimes(); + expect(schDatanode.getServiceComponentName()).andReturn("DATANODE").anyTimes(); + + final ServiceComponentHost schNamenode = createMock(ServiceComponentHost.class); + expect(schNamenode.getServiceName()).andReturn("HDFS").anyTimes(); + expect(schNamenode.getServiceComponentName()).andReturn("NAMENODE").anyTimes(); + + final ServiceComponentHost schHBM = createMock(ServiceComponentHost.class); + expect(schHBM.getServiceName()).andReturn("HBASE").anyTimes(); + expect(schHBM.getServiceComponentName()).andReturn("HBASE_MASTER").anyTimes(); + + // First host contains versionable components + final List<ServiceComponentHost> schsH1 = Arrays.asList(schDatanode, schNamenode); + + // Second host contains versionable components + final List<ServiceComponentHost> schsH2 = Arrays.asList(schDatanode); + + // Third host only has hbase + final List<ServiceComponentHost> schsH3 = Arrays.asList(schHBM); + + ServiceOsSpecific.Package hdfsPackage = new ServiceOsSpecific.Package(); + hdfsPackage.setName("hdfs"); + + List<ServiceOsSpecific.Package> packages = Collections.singletonList(hdfsPackage); + + ActionManager actionManager = createNiceMock(ActionManager.class); + + RequestStatusResponse response = createNiceMock(RequestStatusResponse.class); + ResourceProviderFactory resourceProviderFactory = createNiceMock(ResourceProviderFactory.class); + ResourceProvider csvResourceProvider = createNiceMock(ClusterStackVersionResourceProvider.class); + + AbstractControllerResourceProvider.init(resourceProviderFactory); + + Map<String, Map<String, String>> hostConfigTags = new HashMap<String, Map<String, String>>(); + expect(configHelper.getEffectiveDesiredTags(anyObject(ClusterImpl.class), anyObject(String.class))).andReturn(hostConfigTags); + + expect(managementController.getClusters()).andReturn(clusters).anyTimes(); + expect(managementController.getAmbariMetaInfo()).andReturn(ambariMetaInfo).anyTimes(); + expect(managementController.getAuthName()).andReturn("admin").anyTimes(); + expect(managementController.getActionManager()).andReturn(actionManager).anyTimes(); + expect(managementController.getJdkResourceUrl()).andReturn("/JdkResourceUrl").anyTimes(); + expect(managementController.getPackagesForServiceHost(anyObject(ServiceInfo.class), + (Map<String, String>) anyObject(List.class), anyObject(String.class))). + andReturn(packages).anyTimes(); // only one host has the versionable component + + expect(resourceProviderFactory.getHostResourceProvider(anyObject(Set.class), anyObject(Map.class), + eq(managementController))).andReturn(csvResourceProvider).anyTimes(); + + expect(clusters.getCluster(anyObject(String.class))).andReturn(cluster); + expect(clusters.getHostsForCluster(anyObject(String.class))).andReturn( + hostsForCluster).anyTimes(); + + String clusterName = "Cluster100"; + expect(cluster.getClusterId()).andReturn(1L).anyTimes(); + expect(cluster.getHosts()).andReturn(hostsForCluster.values()).atLeastOnce(); + expect(cluster.getServices()).andReturn(serviceMap).anyTimes(); + expect(cluster.getCurrentStackVersion()).andReturn(stackId); + expect(cluster.getServiceComponentHosts(anyObject(String.class))).andAnswer(new IAnswer<List<ServiceComponentHost>>() { + @Override + public List<ServiceComponentHost> answer() throws Throwable { + String hostname = (String) EasyMock.getCurrentArguments()[0]; + if (hostname.equals("host2")) { + return schsH2; + } else if (hostname.equals("host3")) { + return schsH3; + } else { + return schsH1; + } + } + }).anyTimes(); + +// ExecutionCommand executionCommand = createNiceMock(ExecutionCommand.class); + ExecutionCommand executionCommand = new ExecutionCommand(); + ExecutionCommandWrapper executionCommandWrapper = createNiceMock(ExecutionCommandWrapper.class); + +// expect(executionCommand.getHostLevelParams()).andReturn(new HashMap<String, String>()).atLeastOnce(); + expect(executionCommandWrapper.getExecutionCommand()).andReturn(executionCommand).anyTimes(); + + Stage stage = createNiceMock(Stage.class); + expect(stage.getExecutionCommandWrapper(anyObject(String.class), anyObject(String.class))). + andReturn(executionCommandWrapper).anyTimes(); + + Map<Role, Float> successFactors = new HashMap<>(); + expect(stage.getSuccessFactors()).andReturn(successFactors).atLeastOnce(); + + // Check that we create proper stage count + expect(stageFactory.createNew(anyLong(), anyObject(String.class), + anyObject(String.class), anyLong(), + anyObject(String.class), anyObject(String.class), anyObject(String.class), + anyObject(String.class))).andReturn(stage). + times((int) Math.ceil(hostCount / MAX_TASKS_PER_STAGE)); + + expect( + repositoryVersionDAOMock.findByStackAndVersion( + anyObject(StackId.class), + anyObject(String.class))).andReturn(repoVersion); + + Capture<org.apache.ambari.server.actionmanager.Request> c = Capture.newInstance(); + Capture<ExecuteActionRequest> ear = Capture.newInstance(); + + actionManager.sendActions(capture(c), capture(ear)); + expectLastCall().atLeastOnce(); + expect(actionManager.getRequestTasks(anyLong())).andReturn(Collections.<HostRoleCommand>emptyList()).anyTimes(); + + ClusterEntity clusterEntity = new ClusterEntity(); + clusterEntity.setClusterId(1l); + clusterEntity.setClusterName(clusterName); + ClusterVersionEntity cve = new ClusterVersionEntity(clusterEntity, + repoVersion, RepositoryVersionState.INSTALL_FAILED, 0, ""); + expect(clusterVersionDAO.findByClusterAndStackAndVersion(anyObject(String.class), + anyObject(StackId.class), anyObject(String.class))).andReturn(cve); + + TopologyManager topologyManager = injector.getInstance(TopologyManager.class); + StageUtils.setTopologyManager(topologyManager); + + // replay + replay(managementController, response, clusters, hdfsService, hbaseService, resourceProviderFactory, csvResourceProvider, + cluster, repositoryVersionDAOMock, configHelper, schDatanode, schNamenode, schHBM, actionManager, + executionCommandWrapper,stage, stageFactory, clusterVersionDAO); + + ResourceProvider provider = AbstractControllerResourceProvider.getResourceProvider( + type, + PropertyHelper.getPropertyIds(type), + PropertyHelper.getKeyPropertyIds(type), + managementController); + + injector.injectMembers(provider); + + // add the property map to a set for the request. add more maps for multiple creates + Set<Map<String, Object>> propertySet = new LinkedHashSet<Map<String, Object>>(); + + Map<String, Object> properties = new LinkedHashMap<String, Object>(); + + // add properties to the request map + properties.put(ClusterStackVersionResourceProvider.CLUSTER_STACK_VERSION_CLUSTER_NAME_PROPERTY_ID, "Cluster100"); + properties.put(ClusterStackVersionResourceProvider.CLUSTER_STACK_VERSION_REPOSITORY_VERSION_PROPERTY_ID, "2.2.0.1-885"); + properties.put(ClusterStackVersionResourceProvider.CLUSTER_STACK_VERSION_STACK_PROPERTY_ID, "HDP"); + properties.put(ClusterStackVersionResourceProvider.CLUSTER_STACK_VERSION_VERSION_PROPERTY_ID, "2.1.1"); + + propertySet.add(properties); + + // create the request + Request request = PropertyHelper.getCreateRequest(propertySet, null); + + RequestStatus status = provider.createResources(request); + Assert.assertNotNull(status); + + // verify + verify(managementController, response, clusters, stageFactory, stage); + + // check that the success factor was populated in the stage + Float successFactor = successFactors.get(Role.INSTALL_PACKAGES); + Assert.assertEquals(Float.valueOf(0.85f), successFactor); + + Assert.assertTrue(executionCommand.getRoleParams().containsKey(KeyNames.PACKAGE_VERSION)); + Assert.assertTrue(executionCommand.getRoleParams().containsKey("base_urls")); + Assert.assertEquals("[]", executionCommand.getRoleParams().get("base_urls")); + } + /** * Tests manual finalization scenario * @throws Exception http://git-wip-us.apache.org/repos/asf/ambari/blob/710a8fea/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryVersionResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryVersionResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryVersionResourceProviderTest.java index e031fc8..d07ed76 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryVersionResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryVersionResourceProviderTest.java @@ -518,6 +518,76 @@ public class RepositoryVersionResourceProviderTest { } @Test + public void testUpdateResourcesNoManageRepos() throws Exception { + Authentication authentication = TestAuthenticationFactory.createAdministrator(); + + SecurityContextHolder.getContext().setAuthentication(authentication); + + final ResourceProvider provider = injector.getInstance(ResourceProviderFactory.class).getRepositoryVersionResourceProvider(); + + Mockito.when(clusterVersionDAO.findByStackAndVersion(Mockito.anyString(), Mockito.anyString(), Mockito.anyString())).thenAnswer( + new Answer<List<ClusterVersionEntity>>() { + @Override + public List<ClusterVersionEntity> answer(InvocationOnMock invocation) throws Throwable { + return getNoClusterVersions(); + } + }); + + final Set<Map<String, Object>> propertySet = new LinkedHashSet<Map<String, Object>>(); + final Map<String, Object> properties = new LinkedHashMap<String, Object>(); + properties.put(RepositoryVersionResourceProvider.REPOSITORY_VERSION_DISPLAY_NAME_PROPERTY_ID, "name"); + properties.put(RepositoryVersionResourceProvider.SUBRESOURCE_OPERATING_SYSTEMS_PROPERTY_ID, new Gson().fromJson("[{\"OperatingSystems/os_type\":\"redhat6\",\"repositories\":[{\"Repositories/repo_id\":\"1\",\"Repositories/repo_name\":\"1\",\"Repositories/base_url\":\"http://example.com/repo1\"}]}]";, Object.class)); + properties.put(RepositoryVersionResourceProvider.REPOSITORY_VERSION_STACK_NAME_PROPERTY_ID, "HDP"); + properties.put(RepositoryVersionResourceProvider.REPOSITORY_VERSION_STACK_VERSION_PROPERTY_ID, "1.1"); + properties.put(RepositoryVersionResourceProvider.REPOSITORY_VERSION_REPOSITORY_VERSION_PROPERTY_ID, "1.1.1.1"); + propertySet.add(properties); + + final Predicate predicateStackName = new PredicateBuilder().property(RepositoryVersionResourceProvider.REPOSITORY_VERSION_STACK_NAME_PROPERTY_ID).equals("HDP").toPredicate(); + final Predicate predicateStackVersion = new PredicateBuilder().property(RepositoryVersionResourceProvider.REPOSITORY_VERSION_STACK_VERSION_PROPERTY_ID).equals("1.1").toPredicate(); + final Request getRequest = PropertyHelper.getReadRequest( + RepositoryVersionResourceProvider.REPOSITORY_VERSION_DISPLAY_NAME_PROPERTY_ID, + RepositoryVersionResourceProvider.SUBRESOURCE_OPERATING_SYSTEMS_PROPERTY_ID); + Assert.assertEquals(0, provider.getResources(getRequest, new AndPredicate(predicateStackName, predicateStackVersion)).size()); + + final Request createRequest = PropertyHelper.getCreateRequest(propertySet, null); + provider.createResources(createRequest); + + Assert.assertEquals(1, provider.getResources(getRequest, new AndPredicate(predicateStackName, predicateStackVersion)).size()); + Assert.assertEquals("name", provider.getResources(getRequest, new AndPredicate(predicateStackName, predicateStackVersion)).iterator().next().getPropertyValue(RepositoryVersionResourceProvider.REPOSITORY_VERSION_DISPLAY_NAME_PROPERTY_ID)); + + properties.put(RepositoryVersionResourceProvider.REPOSITORY_VERSION_ID_PROPERTY_ID, "1"); + properties.put(RepositoryVersionResourceProvider.REPOSITORY_VERSION_DISPLAY_NAME_PROPERTY_ID, "name2"); + properties.put(RepositoryVersionResourceProvider.SUBRESOURCE_OPERATING_SYSTEMS_PROPERTY_ID, new Gson().fromJson("[{\"OperatingSystems/ambari_managed_repositories\":false, \"OperatingSystems/os_type\":\"redhat6\",\"repositories\":[{\"Repositories/repo_id\":\"1\",\"Repositories/repo_name\":\"1\",\"Repositories/base_url\":\"http://example.com/repo1\"}]}]";, Object.class)); + final Request updateRequest = PropertyHelper.getUpdateRequest(properties, null); + provider.updateResources(updateRequest, new AndPredicate(predicateStackName, predicateStackVersion)); + + Assert.assertEquals("name2", provider.getResources(getRequest, new AndPredicate(predicateStackName, predicateStackVersion)).iterator().next().getPropertyValue(RepositoryVersionResourceProvider.REPOSITORY_VERSION_DISPLAY_NAME_PROPERTY_ID)); + + AmbariMetaInfo ambariMetaInfo = injector.getInstance(AmbariMetaInfo.class); + String stackName = properties.get(RepositoryVersionResourceProvider.REPOSITORY_VERSION_STACK_NAME_PROPERTY_ID).toString(); + String stackVersion = properties.get(RepositoryVersionResourceProvider.REPOSITORY_VERSION_STACK_VERSION_PROPERTY_ID).toString(); + Object operatingSystems = properties.get(RepositoryVersionResourceProvider.SUBRESOURCE_OPERATING_SYSTEMS_PROPERTY_ID); + Gson gson = new Gson(); + String operatingSystemsJson = gson.toJson(operatingSystems); + RepositoryVersionHelper repositoryVersionHelper = new RepositoryVersionHelper(); + List<OperatingSystemEntity> operatingSystemEntities = repositoryVersionHelper.parseOperatingSystems(operatingSystemsJson); + for (OperatingSystemEntity operatingSystemEntity : operatingSystemEntities) { + Assert.assertFalse(operatingSystemEntity.isAmbariManagedRepos()); + String osType = operatingSystemEntity.getOsType(); + List<RepositoryEntity> repositories = operatingSystemEntity.getRepositories(); + for (RepositoryEntity repository : repositories) { + RepositoryInfo repo = ambariMetaInfo.getRepository(stackName, stackVersion, osType, repository.getRepositoryId()); + if (repo != null) { + String baseUrlActual = repo.getBaseUrl(); + String baseUrlExpected = repository.getBaseUrl(); + Assert.assertEquals(baseUrlExpected, baseUrlActual); + } + } + } + + } + + @Test public void testVersionInStack(){ StackId sid = new StackId("HDP-2.3"); StackId sid2 = new StackId("HDP-2.3.NEW"); http://git-wip-us.apache.org/repos/asf/ambari/blob/710a8fea/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java index 9608b79..8f4e445 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java @@ -83,7 +83,6 @@ import org.apache.ambari.server.state.stack.upgrade.UpgradeType; import org.junit.After; import org.junit.Assert; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import com.google.gson.Gson; @@ -305,7 +304,21 @@ public class UpgradeActionTest { clusters.mapHostsToCluster(Collections.singleton(hostName), clusterName); // Create the starting repo version - m_helper.getOrCreateRepositoryVersion(sourceStack, sourceRepo); + RepositoryVersionEntity repoEntity = m_helper.getOrCreateRepositoryVersion(sourceStack, sourceRepo); + repoEntity.setOperatingSystems("[\n" + + " {\n" + + " \"repositories\":[\n" + + " {\n" + + " \"Repositories/base_url\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0\",\n"; + + " \"Repositories/repo_name\":\"HDP\",\n" + + " \"Repositories/repo_id\":\"HDP-2.2\"\n" + + " }\n" + + " ],\n" + + " \"OperatingSystems/os_type\":\"redhat6\"\n" + + " }\n" + + "]"); + repoVersionDAO.merge(repoEntity); + c.createClusterVersion(sourceStack, sourceRepo, "admin", RepositoryVersionState.INSTALLING); c.transitionClusterVersion(sourceStack, sourceRepo, RepositoryVersionState.CURRENT); http://git-wip-us.apache.org/repos/asf/ambari/blob/710a8fea/ambari-server/src/test/python/custom_actions/TestInstallPackages.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/custom_actions/TestInstallPackages.py b/ambari-server/src/test/python/custom_actions/TestInstallPackages.py index ad4206b..e62978a 100644 --- a/ambari-server/src/test/python/custom_actions/TestInstallPackages.py +++ b/ambari-server/src/test/python/custom_actions/TestInstallPackages.py @@ -120,6 +120,56 @@ class TestInstallPackages(RMFTestCase): self.assertResourceCalled('Package', 'ambari-log4j', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) self.assertNoMoreResources() + @patch("resource_management.libraries.functions.list_ambari_managed_repos.list_ambari_managed_repos") + @patch("resource_management.libraries.functions.packages_analyzer.allInstalledPackages") + @patch("resource_management.libraries.script.Script.put_structured_out") + @patch("resource_management.libraries.functions.stack_select.get_stack_versions") + @patch("resource_management.libraries.functions.repo_version_history.read_actual_version_from_history_file") + @patch("resource_management.libraries.functions.repo_version_history.write_actual_version_to_history_file") + def test_no_repos(self, + write_actual_version_to_history_file_mock, + read_actual_version_from_history_file_mock, + stack_versions_mock, + put_structured_out_mock, allInstalledPackages_mock, list_ambari_managed_repos_mock): + stack_versions_mock.side_effect = [ + [], # before installation attempt + [VERSION_STUB] + ] + + config_file = self.get_src_folder() + "/test/python/custom_actions/configs/install_packages_config.json" + with open(config_file, "r") as f: + command_json = json.load(f) + + command_json['roleParams']['base_urls'] = "[]" + + + allInstalledPackages_mock.side_effect = TestInstallPackages._add_packages + list_ambari_managed_repos_mock.return_value=[] + self.executeScript("scripts/install_packages.py", + classname="InstallPackages", + command="actionexecute", + config_dict = command_json, + target=RMFTestCase.TARGET_CUSTOM_ACTIONS, + os_type=('Redhat', '6.4', 'Final'), + ) + self.assertTrue(put_structured_out_mock.called) + self.assertEquals(put_structured_out_mock.call_args[0][0], + {'package_installation_result': 'SUCCESS', + 'installed_repository_version': VERSION_STUB, + 'stack_id': 'HDP-2.2', + 'actual_version': VERSION_STUB, + 'ambari_repositories': []}) + + self.assertResourceCalled('Package', 'hdp-select', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'hadoop_2_2_0_1_885', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'snappy', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'snappy-devel', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'lzo', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'hadooplzo_2_2_0_1_885', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'hadoop_2_2_0_1_885-libhdfs', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'ambari-log4j', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertNoMoreResources() + @patch("ambari_commons.os_check.OSCheck.is_suse_family") @patch("resource_management.libraries.functions.list_ambari_managed_repos.list_ambari_managed_repos") @patch("resource_management.libraries.functions.packages_analyzer.allInstalledPackages") @@ -922,3 +972,4 @@ class TestInstallPackages(RMFTestCase): 'ambari_repositories': []}) self.assertFalse(write_actual_version_to_history_file_mock.called) + http://git-wip-us.apache.org/repos/asf/ambari/blob/710a8fea/ambari-server/src/test/python/stacks/2.0.6/hooks/before-INSTALL/test_before_install.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/hooks/before-INSTALL/test_before_install.py b/ambari-server/src/test/python/stacks/2.0.6/hooks/before-INSTALL/test_before_install.py index aacd1f2..082d200 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/hooks/before-INSTALL/test_before_install.py +++ b/ambari-server/src/test/python/stacks/2.0.6/hooks/before-INSTALL/test_before_install.py @@ -22,6 +22,7 @@ from mock.mock import MagicMock, call, patch from resource_management import * from stacks.utils.RMFTestCase import * import getpass +import json @patch.object(getpass, "getuser", new = MagicMock(return_value='some_user')) @patch.object(Hook, "run_custom_hook", new = MagicMock()) @@ -43,3 +44,20 @@ class TestHookBeforeInstall(RMFTestCase): self.assertResourceCalled('Package', 'unzip', retry_count=5, retry_on_repo_unavailability=False) self.assertResourceCalled('Package', 'curl', retry_count=5, retry_on_repo_unavailability=False) self.assertNoMoreResources() + + def test_hook_no_repos(self): + + config_file = self.get_src_folder() + "/test/python/stacks/2.0.6/configs/default.json" + with open(config_file, "r") as f: + command_json = json.load(f) + + command_json['hostLevelParams']['repo_info'] = "[]" + + self.executeScript("2.0.6/hooks/before-INSTALL/scripts/hook.py", + classname="BeforeInstallHook", + command="hook", + config_dict=command_json) + + self.assertResourceCalled('Package', 'unzip', retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'curl', retry_count=5, retry_on_repo_unavailability=False) + self.assertNoMoreResources()
