Repository: ambari Updated Branches: refs/heads/branch-feature-AMBARI-14714 7e967d4f7 -> 89bb02c43
AMBARI-21645: Dynamic Reload Stacks (jluniya) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/89bb02c4 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/89bb02c4 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/89bb02c4 Branch: refs/heads/branch-feature-AMBARI-14714 Commit: 89bb02c43ae2130009db656d29f9e8688fdf1153 Parents: 7e967d4 Author: Jayush Luniya <[email protected]> Authored: Thu Aug 3 15:07:45 2017 -0700 Committer: Jayush Luniya <[email protected]> Committed: Thu Aug 3 15:07:45 2017 -0700 ---------------------------------------------------------------------- .../src/main/python/ambari_agent/Controller.py | 14 + .../ambari_agent/CustomServiceOrchestrator.py | 2 + .../ambari/server/agent/HeartBeatHandler.java | 32 ++- .../ambari/server/agent/HeartBeatResponse.java | 7 + .../ambari/server/agent/rest/AgentResource.java | 10 + .../server/api/services/AmbariMetaInfo.java | 11 +- .../server/api/services/StacksService.java | 11 + .../AmbariManagementControllerImpl.java | 5 +- .../ambari/server/stack/StackManager.java | 89 +++++- .../server/stack/StackManagerFactory.java | 10 +- .../server/utils/ResourceFilesKeeper.java | 182 +++++++++++++ .../server/utils/ResourceFilesKeeperHelper.java | 270 +++++++++++++++++++ .../AmbariManagementControllerImplTest.java | 8 +- .../AmbariManagementControllerTest.java | 3 + .../stack/StackManagerCommonServicesTest.java | 6 +- .../server/stack/StackManagerExtensionTest.java | 6 +- .../server/stack/StackManagerMiscTest.java | 24 +- .../ambari/server/stack/StackManagerMock.java | 57 +++- .../ambari/server/stack/StackManagerTest.java | 22 +- .../KerberosDescriptorUpdateHelperTest.java | 5 +- 20 files changed, 721 insertions(+), 53 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/89bb02c4/ambari-agent/src/main/python/ambari_agent/Controller.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/Controller.py b/ambari-agent/src/main/python/ambari_agent/Controller.py index bc923c3..4da631c 100644 --- a/ambari-agent/src/main/python/ambari_agent/Controller.py +++ b/ambari-agent/src/main/python/ambari_agent/Controller.py @@ -94,6 +94,7 @@ class Controller(threading.Thread): self.heartbeat_stop_callback = heartbeat_stop_callback # List of callbacks that are called at agent registration self.registration_listeners = [] + self.refresh_cache_listeners = [] # pull config directory out of config cache_dir = config.get('agent', 'cache_dir') @@ -381,6 +382,14 @@ class Controller(threading.Thread): logger.log(logging_level, "Updating configurations from heartbeat") self.cluster_configuration.update_configurations_from_heartbeat(response) + refreshCache = False + if 'refreshCache' in response.keys(): + refreshCache = bool(response['refreshCache']) + + if refreshCache: + logger.info("Received refreshCache command") + self.refreshCache() + response_keys = response.keys() # there's case when canceled task can be processed in Action Queue.execute before adding rescheduled task to queue @@ -531,6 +540,11 @@ class Controller(threading.Thread): else: logger.info("Registration response from %s didn't contain 'response' as a key".format(self.serverHostname)) + def refreshCache(self): + # Refresh stack + for callback in self.refresh_cache_listeners: + callback() + def restartAgent(self): ExitHelper().exit(AGENT_AUTO_RESTART_EXIT_CODE) http://git-wip-us.apache.org/repos/asf/ambari/blob/89bb02c4/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py index 7dd00de..dec0e91 100644 --- a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py +++ b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py @@ -92,6 +92,8 @@ class CustomServiceOrchestrator(): self.public_fqdn = hostname.public_hostname(config) # cache reset will be called on every agent registration controller.registration_listeners.append(self.file_cache.reset) + # cache reset should also be called on explicit refreshCache command + controller.refresh_cache_listeners.append(self.file_cache.reset) # Construct the hadoop credential lib JARs path self.credential_shell_lib_path = os.path.join(config.get('security', 'credential_lib_dir', http://git-wip-us.apache.org/repos/asf/ambari/blob/89bb02c4/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java index 1bc4c36..435048e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java @@ -23,6 +23,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; @@ -59,6 +60,7 @@ import org.apache.ambari.server.utils.VersionUtils; import org.apache.commons.codec.binary.Base64; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.io.IOUtils; +import org.eclipse.jetty.util.ConcurrentHashSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -107,6 +109,7 @@ public class HeartBeatHandler { private KerberosIdentityDataFileReaderFactory kerberosIdentityDataFileReaderFactory; private Map<String, Long> hostResponseIds = new ConcurrentHashMap<>(); + private Set<String> refreshCacheForHosts = new ConcurrentHashSet<>(); private Map<String, HeartBeatResponse> hostResponses = new ConcurrentHashMap<>(); @@ -126,6 +129,16 @@ public class HeartBeatHandler { heartbeatMonitor.start(); } + public void addRefreshCacheForHosts(List<Host> hosts) { + if(hosts != null && !hosts.isEmpty()) { + List<String> hostNames = new LinkedList<>(); + for(Host host : hosts) { + hostNames.add(host.getHostName()); + } + refreshCacheForHosts.addAll(hostNames); + } + } + void setHeartbeatMonitor(HeartbeatMonitor heartbeatMonitor) { this.heartbeatMonitor = heartbeatMonitor; } @@ -152,7 +165,7 @@ public class HeartBeatHandler { if (currentResponseId == null) { //Server restarted, or unknown host. LOG.error("CurrentResponseId unknown for " + hostname + " - send register command"); - return createRegisterCommand(); + return createRegisterCommand(hostname); } LOG.debug("Received heartbeat from host, hostname={}, currentResponseId={}, receivedResponseId={}", hostname, currentResponseId, heartbeat.getResponseId()); @@ -172,7 +185,7 @@ public class HeartBeatHandler { hostname, currentResponseId); - return createRestartCommand(currentResponseId); + return createRestartCommand(currentResponseId, hostname); } response = new HeartBeatResponse(); @@ -194,7 +207,7 @@ public class HeartBeatHandler { if (hostObject.getState().equals(HostState.HEARTBEAT_LOST)) { // After loosing heartbeat agent should reregister LOG.warn("Host {} is in HEARTBEAT_LOST state - sending register command", hostname); - return createRegisterCommand(); + return createRegisterCommand(hostname); } hostResponseIds.put(hostname, currentResponseId); @@ -227,7 +240,7 @@ public class HeartBeatHandler { } catch (InvalidStateTransitionException ex) { LOG.warn("Asking agent to re-register due to " + ex.getMessage(), ex); hostObject.setState(HostState.INIT); - return createRegisterCommand(); + return createRegisterCommand(hostname); } /* @@ -251,6 +264,11 @@ public class HeartBeatHandler { } } + if(refreshCacheForHosts.contains(hostname)) { + response.setRefreshCache(true); + refreshCacheForHosts.remove(hostname); + } + heartbeatProcessor.addHeartbeat(heartbeat); // Send commands if node is active @@ -344,7 +362,8 @@ public class HeartBeatHandler { return osType.toLowerCase(); } - protected HeartBeatResponse createRegisterCommand() { + protected HeartBeatResponse createRegisterCommand(String hostname) { + refreshCacheForHosts.remove(hostname); HeartBeatResponse response = new HeartBeatResponse(); RegistrationCommand regCmd = new RegistrationCommand(); response.setResponseId(0); @@ -352,7 +371,8 @@ public class HeartBeatHandler { return response; } - protected HeartBeatResponse createRestartCommand(Long currentResponseId) { + protected HeartBeatResponse createRestartCommand(Long currentResponseId, String hostname) { + refreshCacheForHosts.remove(hostname); HeartBeatResponse response = new HeartBeatResponse(); response.setRestartAgent(true); response.setResponseId(currentResponseId); http://git-wip-us.apache.org/repos/asf/ambari/blob/89bb02c4/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatResponse.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatResponse.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatResponse.java index 667a6bf..6e2fdec 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatResponse.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatResponse.java @@ -63,6 +63,9 @@ public class HeartBeatResponse { @SerializedName("restartAgent") private boolean restartAgent = false; + @SerializedName("refreshCache") + private boolean refreshCache = false; + @SerializedName("hasMappedComponents") private boolean hasMappedComponents = false; @@ -162,6 +165,10 @@ public class HeartBeatResponse { this.restartAgent = restartAgent; } + public void setRefreshCache(boolean refreshCache) { + this.refreshCache = refreshCache; + } + public boolean hasMappedComponents() { return hasMappedComponents; } http://git-wip-us.apache.org/repos/asf/ambari/blob/89bb02c4/ambari-server/src/main/java/org/apache/ambari/server/agent/rest/AgentResource.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/rest/AgentResource.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/rest/AgentResource.java index 4f63df9..cd58af8 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/rest/AgentResource.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/rest/AgentResource.java @@ -18,6 +18,8 @@ package org.apache.ambari.server.agent.rest; +import java.util.List; + import javax.servlet.http.HttpServletRequest; import javax.ws.rs.Consumes; import javax.ws.rs.GET; @@ -38,6 +40,7 @@ import org.apache.ambari.server.agent.HeartBeatResponse; import org.apache.ambari.server.agent.Register; import org.apache.ambari.server.agent.RegistrationResponse; import org.apache.ambari.server.agent.RegistrationStatus; +import org.apache.ambari.server.state.Host; import org.apache.ambari.server.state.fsm.InvalidStateTransitionException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -69,6 +72,13 @@ public class AgentResource { } /** + * Explicitly refresh cache for a host + */ + public static void addRefreshCacheForHosts(List<Host> hosts) { + hh.addRefreshCacheForHosts(hosts); + } + + /** * Register information about the host (Internal API to be used for * Ambari Agent) * @response.representation.200.doc This API is invoked by Ambari agent running http://git-wip-us.apache.org/repos/asf/ambari/blob/89bb02c4/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java index 6e4b5fa..46beb45 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java @@ -162,6 +162,7 @@ public class AmbariMetaInfo { private final ActionDefinitionManager adManager = new ActionDefinitionManager(); private String serverVersion = "undefined"; + private File resourcesRoot; private File stackRoot; private File commonServicesRoot; private File extensionsRoot; @@ -251,6 +252,12 @@ public class AmbariMetaInfo { @Inject public AmbariMetaInfo(Configuration conf) throws Exception { this.conf = conf; + + String resourcesPath = conf.getResourceDirPath(); + if(resourcesPath != null && !resourcesPath.isEmpty()) { + resourcesRoot = new File(resourcesPath); + } + String stackPath = conf.getMetadataPath(); stackRoot = new File(stackPath); @@ -286,8 +293,8 @@ public class AmbariMetaInfo { readServerVersion(); - stackManager = stackManagerFactory.create(stackRoot, commonServicesRoot, extensionsRoot, - osFamily, false); + stackManager = stackManagerFactory.create(resourcesRoot, stackRoot, commonServicesRoot, extensionsRoot, + osFamily, false /* validate = false */, true /* refreshArchives = true */); mpackManager = mpackManagerFactory.create(mpacksV2Staging, stackRoot); http://git-wip-us.apache.org/repos/asf/ambari/blob/89bb02c4/ambari-server/src/main/java/org/apache/ambari/server/api/services/StacksService.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/StacksService.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/StacksService.java index 9c50821..67d3a15 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/StacksService.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/StacksService.java @@ -23,6 +23,7 @@ import java.util.HashMap; import java.util.Map; import javax.ws.rs.GET; +import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; @@ -84,6 +85,16 @@ public class StacksService extends BaseService { createStackResource(null)); } + @PUT + @Produces("text/plain") + @ApiOperation(value = "Reload all stacks", + nickname = "StacksService#reloadStacks") + public Response reloadStacks(@Context HttpHeaders headers, @Context UriInfo ui) { + + return handleRequest(headers, null, ui, Request.Type.PUT, + createStackResource(null)); + } + @GET @Path("{stackName}") @Produces(MediaType.TEXT_PLAIN) http://git-wip-us.apache.org/repos/asf/ambari/blob/89bb02c4/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index 0f45270..aeefa55 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -89,6 +89,7 @@ import org.apache.ambari.server.actionmanager.Stage; import org.apache.ambari.server.actionmanager.StageFactory; import org.apache.ambari.server.agent.ExecutionCommand; import org.apache.ambari.server.agent.ExecutionCommand.KeyNames; +import org.apache.ambari.server.agent.rest.AgentResource; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.api.services.LoggingService; import org.apache.ambari.server.configuration.Configuration; @@ -4427,14 +4428,16 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle public synchronized RequestStatusResponse updateStacks() throws AmbariException { try { + // Refresh stacks API endpoint and refresh archives ambariMetaInfo.init(); + // Add "refreshCache" command in the next agent hearbeat for all hosts + AgentResource.addRefreshCacheForHosts(clusters.getHosts()); } catch (AmbariException e) { throw e; } catch (Exception e) { throw new AmbariException( "Ambari Meta Information can't be read from the stack root directory"); } - return null; } http://git-wip-us.apache.org/repos/asf/ambari/blob/89bb02c4/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java index b498fcb..32ac270 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java @@ -51,6 +51,7 @@ import org.apache.ambari.server.state.ServiceInfo; import org.apache.ambari.server.state.StackInfo; import org.apache.ambari.server.state.stack.OsFamily; import org.apache.ambari.server.state.stack.ServiceMetainfoXml; +import org.apache.ambari.server.utils.ResourceFilesKeeperHelper; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,6 +60,7 @@ import org.xml.sax.SAXException; import com.google.inject.assistedinject.Assisted; import com.google.inject.assistedinject.AssistedInject; + /** * Manages all stack related behavior including parsing of stacks and providing access to * stack information. @@ -66,6 +68,10 @@ import com.google.inject.assistedinject.AssistedInject; public class StackManager { public static final String PROPERTY_SCHEMA_PATH = "configuration-schema.xsd"; + private static final String CUSTOM_ACTIONS_DIR="custom_actions"; + private static final String HOST_SCRIPTS_DIR="host_scripts"; + private static final String DASHBOARDS_DIR="dashboards"; + /** * Delimiter used for parent path string * Example: @@ -91,7 +97,6 @@ public class StackManager { */ private StackContext stackContext; - private File stackRoot; /** * Logger @@ -116,7 +121,9 @@ public class StackManager { /** * Constructor. Initialize stack manager. * - * @param stackRootDir + * @param resourcesRoot + * resources root directory + * @param stackRoot * stack root directory * @param commonServicesRoot * common services root directory @@ -124,6 +131,10 @@ public class StackManager { * extensions root directory * @param osFamily * the OS family read from resources + * @param validate + * validate all stack and service definitions + * @param refreshArchives + * refresh archive.zip and .hash * @param metaInfoDAO * metainfo DAO automatically injected * @param actionMetadata @@ -141,17 +152,18 @@ public class StackManager { * if an exception occurs while processing the stacks */ @AssistedInject - public StackManager(@Assisted("stackRoot") File stackRootDir, + public StackManager(@Assisted("resourcesRoot") File resourcesRoot, + @Assisted("stackRoot") File stackRoot, @Assisted("commonServicesRoot") @Nullable File commonServicesRoot, @Assisted("extensionRoot") @Nullable File extensionRoot, - @Assisted OsFamily osFamily, @Assisted boolean validate, + @Assisted OsFamily osFamily, @Assisted("validate") boolean validate, + @Assisted("refreshArchives") boolean refreshArchives, MetainfoDAO metaInfoDAO, ActionMetadata actionMetadata, StackDAO stackDao, ExtensionDAO extensionDao, ExtensionLinkDAO linkDao, AmbariManagementHelper helper) throws AmbariException { LOG.info("Initializing the stack manager..."); - stackRoot = stackRootDir; if (validate) { validateStackDirectory(stackRoot); validateCommonServicesDirectory(commonServicesRoot); @@ -187,9 +199,76 @@ public class StackManager { fullyResolveExtensions(stackModules, commonServiceModules, extensionModules); fullyResolveStacks(stackModules, commonServiceModules, extensionModules); + if(refreshArchives) { + updateArchives(resourcesRoot, stackRoot, stackModules, commonServiceModules, extensionModules); + } + populateDB(stackDao, extensionDao); } + protected void updateArchives( + File resourcesRoot, File stackRoot, Map<String, StackModule> stackModules, Map<String, ServiceModule> commonServiceModules, + Map<String, ExtensionModule> extensionModules ) throws AmbariException { + + LOG.info("Refreshing archives ..."); + + LOG.debug("Refreshing archives for stacks"); + for (StackModule stackModule : stackModules.values()) { + LOG.debug("Refreshing archives for stack : " + stackModule.getId()); + String hooksDir = stackModule.getStackDirectory().getHooksDir(); + if(hooksDir != null) { + LOG.debug("Refreshing archive for stack hooks directory : " + hooksDir); + String hooksAbsolutePath = stackRoot.getAbsolutePath() + File.separator + hooksDir; + ResourceFilesKeeperHelper.updateDirectoryArchive(hooksAbsolutePath, false); + } + for(ServiceModule serviceModule : stackModule.getServiceModules().values()) { + String packageDir = serviceModule.getServiceDirectory().getPackageDir(); + if(packageDir != null) { + LOG.debug("Refreshing archive for stack service package directory : " + packageDir); + String packageAbsoluteDir = resourcesRoot.getAbsolutePath() + File.separator + packageDir; + ResourceFilesKeeperHelper.updateDirectoryArchive(packageAbsoluteDir, false); + } + } + } + + LOG.debug("Refreshing archives for common services"); + for(ServiceModule serviceModule : commonServiceModules.values()) { + String packageDir = serviceModule.getServiceDirectory().getPackageDir(); + if(packageDir != null) { + LOG.debug("Refreshing archive for common service package directory : " + packageDir); + String packageAbsoluteDir = resourcesRoot.getAbsolutePath() + File.separator + packageDir; + ResourceFilesKeeperHelper.updateDirectoryArchive(packageAbsoluteDir, false); + } + } + + LOG.debug("Refreshing archives for extensions"); + for(ExtensionModule extensionModule : extensionModules.values()) { + LOG.debug("Refreshing archives for extension module" + extensionModule.getId()); + for(ServiceModule serviceModule : extensionModule.getServiceModules().values()) { + String packageDir = serviceModule.getServiceDirectory().getPackageDir(); + if(packageDir != null) { + LOG.debug("Refreshing archive for extension service package directory : " + packageDir); + String packageAbsoluteDir = resourcesRoot.getAbsolutePath() + File.separator + packageDir; + ResourceFilesKeeperHelper.updateDirectoryArchive(packageAbsoluteDir, false); + } + } + } + + List<String> miscDirs = new ArrayList<String>() {{ + add(CUSTOM_ACTIONS_DIR); + add(HOST_SCRIPTS_DIR); + add(DASHBOARDS_DIR); + }}; + + LOG.debug("Refreshing archives for misc directories"); + for(String miscDir : miscDirs) { + LOG.debug("Refreshing archive for misc directory : " + miscDir); + String miscAbsolutePath = resourcesRoot.getAbsolutePath() + File.separator + miscDir; + ResourceFilesKeeperHelper.updateDirectoryArchive(miscAbsolutePath, false); + } + LOG.info("Refreshing archives finished!"); + } + protected void parseDirectories(File stackRoot, File commonServicesRoot, File extensionRoot) throws AmbariException { commonServiceModules = parseCommonServicesDirectory(commonServicesRoot); stackModules = parseStackDirectory(stackRoot); http://git-wip-us.apache.org/repos/asf/ambari/blob/89bb02c4/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManagerFactory.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManagerFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManagerFactory.java index 8bce4f8..9f8c67c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManagerFactory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManagerFactory.java @@ -33,6 +33,8 @@ import com.google.inject.assistedinject.AssistedInject; public interface StackManagerFactory { /** + * @param resourcesRoot + * the root of resources (not {@code null}). * @param stackRoot * the root of the stack (not {@code null}). * @param commonServicesRoot @@ -42,10 +44,14 @@ public interface StackManagerFactory { * the root of the extensions (not {@code null}). * @param osFamily * the list of all parsed OS families (not {@code null}). + * @param validate + * validate all stack and service definitions + * @param refreshArchives + * refresh archive.zip and .hash * @return a stack manager instance which contains all parsed stacks. */ - StackManager create(@Assisted("stackRoot") File stackRoot, + StackManager create(@Assisted("resourcesRoot") File resourcesRoot, @Assisted("stackRoot") File stackRoot, @Nullable @Assisted("commonServicesRoot") File commonServicesRoot, @Assisted("extensionRoot") @Nullable File extensionRoot, - OsFamily osFamily, boolean validate); + OsFamily osFamily, @Assisted("validate") boolean validate, @Assisted("refreshArchives") boolean refreshArchives); } http://git-wip-us.apache.org/repos/asf/ambari/blob/89bb02c4/ambari-server/src/main/java/org/apache/ambari/server/utils/ResourceFilesKeeper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/utils/ResourceFilesKeeper.java b/ambari-server/src/main/java/org/apache/ambari/server/utils/ResourceFilesKeeper.java new file mode 100644 index 0000000..0d17821 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/ResourceFilesKeeper.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.utils; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +import org.apache.ambari.server.AmbariException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ResourceFilesKeeper { + private static final Logger LOG = LoggerFactory.getLogger + (ResourceFilesKeeper.class); + + private static final String HOOKS_DIR = "hooks"; + private static final String PACKAGE_DIR = "package"; + private static final String COMMON_SERVICES_DIR = "common-services"; + private static final String EXTENSIONS_DIR = "extensions"; + private static final String CUSTOM_ACTIONS_DIR="custom_actions"; + private static final String HOST_SCRIPTS_DIR="host_scripts"; + private static final String DASHBOARDS_DIR="dashboards"; + + private static final String METAINFO_XML = "metainfo.xml"; + + private String resourcesDir; + private String stacksRoot; + private boolean noZip; + private boolean verbose; + + private List<String> archive_directories = new ArrayList<String>() {{ + add(HOOKS_DIR); + add(PACKAGE_DIR); + }}; + + public ResourceFilesKeeper() { + } + + public void setNoZip(boolean noZip) { + this.noZip = noZip; + } + + public void setResourcesDir(String resourcesDir) { + this.resourcesDir = resourcesDir; + } + + public void setStacksRoot(String stacksRoot) { + this.stacksRoot = stacksRoot; + } + + public void setVerbose(boolean verbose) { + this.verbose = verbose; + } + + public void updateAllDirectoryArchives() { + List<String> validStacks = getValidStacks(); + iterativeUpdateArchiveDirectories(validStacks); + + String commonServicesRoot = resourcesDir + File.separator + COMMON_SERVICES_DIR; + List<String> validCommonServices = getValidCommonServices(commonServicesRoot); + iterativeUpdateArchiveDirectories(validCommonServices); + + String extensionsRoot = resourcesDir + File.separator + EXTENSIONS_DIR; + List<String> validValidExtensions = getValidExtensions(extensionsRoot); + iterativeUpdateArchiveDirectories(validValidExtensions); + + updateResourcesSubDirArchive(CUSTOM_ACTIONS_DIR); + + updateResourcesSubDirArchive(HOST_SCRIPTS_DIR); + + updateResourcesSubDirArchive(DASHBOARDS_DIR); + } + + public List<String> getValidStacks() { + return getMetainfoDirectories(stacksRoot); + } + + public List<String> getValidCommonServices(String commonServicesRoot) { + return getMetainfoDirectories(commonServicesRoot); + } + + public List<String> getValidExtensions(String extensionsRoot) { + return getMetainfoDirectories(extensionsRoot); + } + + public void updateResourcesSubDirArchive(String subDir) { + String fullPath = resourcesDir + File.separator + subDir; + File fullPathFile = new File(fullPath); + if (fullPathFile.exists() && fullPathFile.isDirectory()) { + ResourceFilesKeeperHelper.updateDirectoryArchive(fullPath, noZip); + } + } + + public void iterativeUpdateArchiveDirectories(List<String> subDirsList) { + for (String subDir : subDirsList) { + File[] serviceFiles = new File(subDir).listFiles(); + for (File serviceFile : serviceFiles) { + if (serviceFile.isDirectory()) { + if (archive_directories.contains(serviceFile.getName())) { + ResourceFilesKeeperHelper.updateDirectoryArchive(serviceFile.getPath(), noZip); + } + } + } + } + } + + public List<String> getMetainfoDirectories(String rootDir) { + List<String> validItems = new ArrayList<>(); + List<File> metainfoDirectories = new ArrayList<>(); + File rootStackDir = new File(rootDir); + + if (!rootStackDir.exists() || !rootStackDir.isDirectory()) { + return new ArrayList<>(); + } + + for (File file : rootStackDir.listFiles()) { + if (file.isDirectory()) { + for (File stackFile : file.listFiles()) { + if (stackFile.isDirectory()) { + metainfoDirectories.add(stackFile); + } + } + } + } + + for (File stackDir : metainfoDirectories) { + String metainfoPath = stackDir.getPath() + File.separator + METAINFO_XML; + if (new File(metainfoPath).exists()) { + validItems.add(stackDir.getPath()); + } + } + return validItems; + } + + + /* + * Main method from which we are calling all checks + * */ + public static void main(String[] args) throws Exception { + ResourceFilesKeeper resourceFilesKeeper = null; + try { + + resourceFilesKeeper = new ResourceFilesKeeper(); + resourceFilesKeeper.setResourcesDir(args[0]); + resourceFilesKeeper.setStacksRoot(args[1]); + resourceFilesKeeper.setNoZip(Boolean.parseBoolean(args[2])); + //resourceFilesKeeperHelper.setVerbose(Boolean.parseBoolean(args[3])); + + System.out.println("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA!!!!"); + + resourceFilesKeeper.updateAllDirectoryArchives(); + + } catch (Throwable e) { + if (e instanceof AmbariException) { + LOG.error("Exception occurred during updating archives:", e); + throw (AmbariException)e; + } else { + LOG.error("Unexpected error, updating archives failed", e); + throw new Exception("Unexpected error, updating archives failed", e); + } + } + } + + +} + http://git-wip-us.apache.org/repos/asf/ambari/blob/89bb02c4/ambari-server/src/main/java/org/apache/ambari/server/utils/ResourceFilesKeeperHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/utils/ResourceFilesKeeperHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/utils/ResourceFilesKeeperHelper.java new file mode 100644 index 0000000..e1aff9a --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/ResourceFilesKeeperHelper.java @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.utils; + + +import java.io.BufferedReader; +import java.io.DataInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ResourceFilesKeeperHelper { + + private static final String HASH_SUM_FILE=".hash"; + private static final String ARCHIVE_NAME="archive.zip"; + private static final String PYC_EXT=".pyc"; + + private static int BUFFER_SIZE = 1024 * 32; + + private static final Logger LOG = LoggerFactory.getLogger + (ResourceFilesKeeperHelper.class); + + /** + * Refresh directory hash and archive if the contents of the directory have changed + * @param directory the directory whose archive needs to be updated + * @param noZip If set, updates only the hash file and skips creating the archive file + */ + public static void updateDirectoryArchive(String directory, boolean noZip) { + boolean skipEmptyDirectory = true; + File dir = new File(directory); + + String newHash = calcHashSum(directory); + String oldHash = readHashSum(directory); + + LOG.info("Directory {} :: oldHash = {}, newHash = {}", directory, oldHash, newHash); + + if (!newHash.equals(oldHash)) { + if (!noZip) { + LOG.info("Creating archive for directory " + directory); + zipDirectory(directory, skipEmptyDirectory); + } + if (skipEmptyDirectory && (!dir.exists() && dir.listFiles().length == 0)) { + LOG.info("Empty directory. Skipping generation of hash file for " + directory); + } else { + writeHashSum(directory, newHash); + } + } else if (!dir.isFile()) { + zipDirectory(directory, skipEmptyDirectory); + } + } + + /** + * Calculate hash on the specified directory + * @param directory the directory for which hash needs to be calculated + * @return calculated hash sum + */ + private static String calcHashSum(String directory) { + + File folder = new File(directory); + if (!folder.isDirectory()) { + return ""; + } + Collection<File> listOfFilesInFolder = FileUtils.listFiles(folder, null, true); + List<String> sortedListOfFilePaths = new ArrayList<>(); + MessageDigest digest = null; + try { + digest = MessageDigest.getInstance("SHA-1"); + } catch (NoSuchAlgorithmException e) { + } + + for (File file : listOfFilesInFolder) { + if (!isFileIgnored(file)) { + sortedListOfFilePaths.add(file.getPath()); + } + } + Collections.sort(sortedListOfFilePaths); + for (String filePath : sortedListOfFilePaths) { + InputStream fis = null; + try { + fis = new FileInputStream(filePath); + int n = 0; + byte[] buffer = new byte[BUFFER_SIZE]; + while (n != -1) { + n = fis.read(buffer); + if (n > 0) { + digest.update(buffer, 0, n); + } + } + } catch (FileNotFoundException e) { + } catch (IOException e) { + } finally { + try { + fis.close(); + } catch (IOException e) { + } + } + + } + return new String(Hex.encodeHex(digest.digest())); + } + + /** + * Read hash value from the hash file in the specified directory + * @param directory the directory from where to read the hash value + * @return the hash value read from the hash file + */ + private static String readHashSum(String directory) { + String pathToHash = directory + File.separator + HASH_SUM_FILE; + File hashFile = new File(pathToHash); + String hash = ""; + + if (hashFile.exists() && !hashFile.isDirectory()) { + InputStream fis = null; + try { + fis = new FileInputStream(pathToHash); + DataInputStream in = new DataInputStream(fis); + BufferedReader br = new BufferedReader(new InputStreamReader(in)); + hash = br.readLine(); + } catch (FileNotFoundException e) { + } catch (IOException e) { + } finally { + try { + fis.close(); + } catch (IOException e) { + } + } + } + return hash == null ? "" : hash.trim(); + } + + /** + * Write hash value to the hash file in the specified directory + * @param directory directory with corresponding hash + * @param hash hash value to be written + */ + private static void writeHashSum(String directory, String hash) { + + String hashFilePath = directory + File.separator + HASH_SUM_FILE; + + PrintWriter out = null; + try { + out = new PrintWriter(hashFilePath); + out.print(hash); + + File hashFile = new File(hashFilePath); + hashFile.setExecutable(true, false); + hashFile.setReadable(true, false); + hashFile.setWritable(true, true); + } catch (FileNotFoundException e) { + } finally { + out.close(); + } + } + + /** + * Create archive zip file for specified directory + * @param directory the directory that needs to be archived + * @param skipEmptyDirectory If set skips creating archives for empty directories + */ + private static void zipDirectory(String directory, boolean skipEmptyDirectory) { + File dirFile = new File(directory); + if (!dirFile.exists() || (skipEmptyDirectory && dirFile.listFiles().length == 0)) { + LOG.info("Empty or non existing directory. Skipping archive creation for " + directory); + return; + } + + String zipFilePath = directory + File.separator + ARCHIVE_NAME; + Collection<File> filesInDirectory = FileUtils.listFiles(dirFile, null, true); + + ZipOutputStream zos = null; + FileOutputStream fos = null; + try { + byte[] buffer = new byte[1024]; + + fos = new FileOutputStream(zipFilePath); + zos = new ZipOutputStream(fos); + + for (File file : filesInDirectory) { + if (!isFileIgnored(file)) { + FileInputStream fis = null; + try{ + fis = new FileInputStream(file); + + // begin writing a new ZIP entry, positions the stream to the start of the entry data + String filePathInZip = file.getPath().replaceAll(dirFile.getPath(), ""); + filePathInZip = filePathInZip.startsWith(File.separator) ? + filePathInZip.replaceFirst(File.separator, "") : filePathInZip; + zos.putNextEntry(new ZipEntry(filePathInZip)); + + int length; + while ((length = fis.read(buffer)) > 0) { + zos.write(buffer, 0, length); + } + + zos.closeEntry(); + } finally { + if (fis != null) { + // close the InputStream + fis.close(); + } + } + + } + } + + } + catch (IOException e) { + } + finally { + try { + if (zos != null) { + zos.close(); + } + if (fos != null) { + fos.close(); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + File archiveFile = new File(zipFilePath); + archiveFile.setExecutable(true, false); + archiveFile.setReadable(true, false); + archiveFile.setWritable(true, true); + } + + /** + * Check if a file should be ignored when computed the hash for a directory + * @param file the file to be checked + * @return True if file should ignore, otherwise False + */ + private static boolean isFileIgnored(File file) { + String fileName = file.getName(); + return (fileName.equals(HASH_SUM_FILE) || fileName.equals(ARCHIVE_NAME) || fileName.endsWith(PYC_EXT) || file.isDirectory()); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/89bb02c4/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java index 6e66e3d..f5921a1 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java @@ -71,6 +71,8 @@ import org.apache.ambari.server.ServiceNotFoundException; import org.apache.ambari.server.actionmanager.ActionDBAccessorImpl; import org.apache.ambari.server.actionmanager.ActionManager; +import org.apache.ambari.server.agent.HeartBeatHandler; +import org.apache.ambari.server.agent.rest.AgentResource; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.internal.RequestStageContainer; @@ -2429,14 +2431,16 @@ public class AmbariManagementControllerImplTest { mpack.setName("testMpack"); MpackResponse mpackResponse = new MpackResponse(mpack); Injector injector = createNiceMock(Injector.class); + AgentResource.init(createNiceMock(HeartBeatHandler.class)); expect(injector.getInstance(MaintenanceStateHelper.class)).andReturn(null).atLeastOnce(); + expect(clusters.getHosts()).andReturn(Collections.emptyList()).anyTimes(); expect(ambariMetaInfo.registerMpack(mpackRequest)).andReturn(mpackResponse); ambariMetaInfo.init(); expectLastCall(); - replay(ambariMetaInfo,injector); + replay(ambariMetaInfo, injector); AmbariManagementController controller = new AmbariManagementControllerImpl(null, clusters, injector); setAmbariMetaInfo(ambariMetaInfo, controller); - Assert.assertEquals(mpackResponse,controller.registerMpack(mpackRequest)); + Assert.assertEquals(mpackResponse, controller.registerMpack(mpackRequest)); } @Test http://git-wip-us.apache.org/repos/asf/ambari/blob/89bb02c4/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java index fdfca0f..c26997d 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java @@ -73,6 +73,8 @@ import org.apache.ambari.server.actionmanager.Stage; import org.apache.ambari.server.actionmanager.StageFactory; import org.apache.ambari.server.actionmanager.TargetHostType; import org.apache.ambari.server.agent.ExecutionCommand; +import org.apache.ambari.server.agent.HeartBeatHandler; +import org.apache.ambari.server.agent.rest.AgentResource; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.audit.AuditLogger; import org.apache.ambari.server.configuration.Configuration; @@ -7914,6 +7916,7 @@ public class AmbariManagementControllerTest { assertTrue(INCORRECT_BASE_URL.equals(repositoryInfo.getBaseUrl())); } + AgentResource.init(createNiceMock(HeartBeatHandler.class)); stackManagerMock.invalidateCurrentPaths(); controller.updateStacks(); http://git-wip-us.apache.org/repos/asf/ambari/blob/89bb02c4/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerCommonServicesTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerCommonServicesTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerCommonServicesTest.java index 5ee8844..8dcb459 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerCommonServicesTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerCommonServicesTest.java @@ -126,8 +126,10 @@ public class StackManagerCommonServicesTest { replay(metaInfoDao, actionMetadata); AmbariManagementHelper helper = new AmbariManagementHelper(stackDao, extensionDao, linkDao); - StackManager stackManager = new StackManager(new File(stackRoot), new File( - commonServicesRoot), new File(extensionRoot), osFamily, true, metaInfoDao, + File stacksRoot = new File(stackRoot); + File resourcesRoot = stacksRoot.getParentFile(); + StackManager stackManager = new StackManager(resourcesRoot, stacksRoot, new File( + commonServicesRoot), new File(extensionRoot), osFamily, true, false, metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao, helper); EasyMock.verify( config, stackDao ); http://git-wip-us.apache.org/repos/asf/ambari/blob/89bb02c4/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerExtensionTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerExtensionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerExtensionTest.java index 20c8f40..2bc4e2c 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerExtensionTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerExtensionTest.java @@ -95,6 +95,8 @@ public class StackManagerExtensionTest { replay(actionMetadata, stackDao, metaInfoDao, osFamily, extensionDao, linkDao); //linkEntity String stacks = ClassLoader.getSystemClassLoader().getResource("stacks_with_extensions").getPath(); + File stacksRoot = new File(stacks); + File resourcesRoot = stacksRoot.getParentFile(); String common = ClassLoader.getSystemClassLoader().getResource("common-services").getPath(); String extensions = ClassLoader.getSystemClassLoader().getResource("extensions").getPath(); @@ -102,8 +104,8 @@ public class StackManagerExtensionTest { StackManager stackManager = null; try { - stackManager = new StackManager(new File(stacks), - new File(common), new File(extensions), osFamily, false, + stackManager = new StackManager(resourcesRoot, stacksRoot, + new File(common), new File(extensions), osFamily, false, false, metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao, helper); } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/ambari/blob/89bb02c4/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMiscTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMiscTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMiscTest.java index 5a5cafa..5e5ef0c 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMiscTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMiscTest.java @@ -74,8 +74,9 @@ public class StackManagerMiscTest { try { String stacksCycle1 = ClassLoader.getSystemClassLoader().getResource("stacks_with_cycle").getPath(); - - StackManager stackManager = new StackManager(new File(stacksCycle1), null, null, osFamily, false, + File stacksRoot = new File(stacksCycle1); + File resourcesRoot = new File(stacksRoot.getParent()); + StackManager stackManager = new StackManager(resourcesRoot, stacksRoot, null, null, osFamily, false, false, metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao, helper); fail("Expected exception due to cyclic stack"); @@ -86,9 +87,10 @@ public class StackManagerMiscTest { try { String stacksCycle2 = ClassLoader.getSystemClassLoader().getResource( "stacks_with_cycle2").getPath(); - - StackManager stackManager = new StackManager(new File(stacksCycle2), - null, null, osFamily, true, metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao, helper); + File stacksRoot = new File(stacksCycle2); + File resourcesRoot = new File(stacksRoot.getParent()); + StackManager stackManager = new StackManager(resourcesRoot, stacksRoot, + null, null, osFamily, true, false, metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao, helper); fail("Expected exception due to cyclic stack"); } catch (AmbariException e) { @@ -128,9 +130,11 @@ public class StackManagerMiscTest { String singleStack = ClassLoader.getSystemClassLoader().getResource("single_stack").getPath(); AmbariManagementHelper helper = new AmbariManagementHelper(stackDao, extensionDao, linkDao); - StackManager stackManager = new StackManager(new File(singleStack.replace( - StackManager.PATH_DELIMITER, File.separator)), null, null, osFamily, false, metaInfoDao, - actionMetadata, stackDao, extensionDao, linkDao, helper); + File stacksRoot = new File(singleStack.replace(StackManager.PATH_DELIMITER, File.separator)); + File resourcesRoot = new File(stacksRoot.getParent()); + + StackManager stackManager = new StackManager(resourcesRoot, stacksRoot, null, null, osFamily, false, false, + metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao, helper); Collection<StackInfo> stacks = stackManager.getStacks(); assertEquals(1, stacks.size()); @@ -168,8 +172,10 @@ public class StackManagerMiscTest { try { String upgradeCycle = ClassLoader.getSystemClassLoader().getResource("stacks_with_upgrade_cycle").getPath(); + File stacksRoot = new File(upgradeCycle); + File resourcesRoot = new File(stacksRoot.getParent()); - StackManager stackManager = new StackManager(new File(upgradeCycle), null, null, osFamily, false, + StackManager stackManager = new StackManager(resourcesRoot, stacksRoot, null, null, osFamily, false, false, metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao, helper); fail("Expected exception due to cyclic service upgrade xml"); http://git-wip-us.apache.org/repos/asf/ambari/blob/89bb02c4/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMock.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMock.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMock.java index 1a61804..209f081 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMock.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMock.java @@ -36,8 +36,8 @@ import org.apache.ambari.server.orm.dao.StackDAO; import org.apache.ambari.server.state.StackInfo; import org.apache.ambari.server.state.stack.OsFamily; -import com.google.inject.Inject; import com.google.inject.assistedinject.Assisted; +import com.google.inject.assistedinject.AssistedInject; /** * Directory tree rescans and stack modules parsing take much time on every module init. @@ -51,6 +51,44 @@ public class StackManagerMock extends StackManager { // Some tests use different stack locations. private static final Map<ModulesPathsKey, CachedModules> pathsToCachedModulesMap = new HashMap<>(); + /** + * Constructor. Initialize stack manager. + * + * @param resourcesRoot resources root directory + * @param stackRoot stack root directory + * @param commonServicesRoot common services root directory + * @param extensionRoot extensions root directory + * @param osFamily the OS family read from resources + * @param validate validate all stack and service definitions + * @param refreshArchives refresh archive.zip and .hash + * @param metaInfoDAO metainfo DAO automatically injected + * @param actionMetadata action meta data automatically injected + * @param stackDao stack DAO automatically injected + * @param extensionDao extension DAO automatically injected + * @param linkDao extension link DAO automatically injected + * @param helper Ambari management helper automatically injected + * @throws AmbariException if an exception occurs while processing the stacks + */ + @AssistedInject + public StackManagerMock( + @Assisted("resourcesRoot") final File resourcesRoot, + @Assisted("stackRoot") final File stackRoot, + @Assisted("commonServicesRoot") @Nullable final File commonServicesRoot, + @Assisted("extensionRoot") @Nullable final File extensionRoot, + @Assisted final OsFamily osFamily, + @Assisted("validate") final boolean validate, + @Assisted("refreshArchives") final boolean refreshArchives, + final MetainfoDAO metaInfoDAO, + final ActionMetadata actionMetadata, + final StackDAO stackDao, + final ExtensionDAO extensionDao, + final ExtensionLinkDAO linkDao, final AmbariManagementHelper helper) throws AmbariException { + super(resourcesRoot, stackRoot, commonServicesRoot, extensionRoot, osFamily, validate, refreshArchives, metaInfoDAO, actionMetadata, stackDao, extensionDao, linkDao, helper); + currentStackRoot = stackRoot; + currentCommonServicesRoot = commonServicesRoot; + currentExtensionRoot = extensionRoot; + } + public static void invalidateKey(File stackRoot, File commonServicesRoot, File extensionRoot) { ModulesPathsKey pathsKey = new ModulesPathsKey(stackRoot, commonServicesRoot, extensionRoot); pathsToCachedModulesMap.remove(pathsKey); @@ -130,16 +168,13 @@ public class StackManagerMock extends StackManager { } } - @Inject - public StackManagerMock(@Assisted("stackRoot") File stackRoot, @Nullable @Assisted("commonServicesRoot") - File commonServicesRoot, @Assisted("extensionRoot") @Nullable File extensionRoot, - @Assisted OsFamily osFamily, @Assisted boolean validate, MetainfoDAO metaInfoDAO, - ActionMetadata actionMetadata, StackDAO stackDao, ExtensionDAO extensionDao, - ExtensionLinkDAO linkDao, AmbariManagementHelper helper) throws AmbariException { - super(stackRoot, commonServicesRoot, extensionRoot, osFamily, validate, metaInfoDAO, actionMetadata, stackDao, extensionDao, linkDao, helper); - currentStackRoot = stackRoot; - currentCommonServicesRoot = commonServicesRoot; - currentExtensionRoot = extensionRoot; + @Override + protected void updateArchives( + File resourcesRoot, File stackRoot, Map<String, StackModule> stackModules, Map<String, ServiceModule> commonServiceModules, + Map<String, ExtensionModule> extensionModules ) throws AmbariException { + /* + * Note: Skip refreshing archives + */ } @Override http://git-wip-us.apache.org/repos/asf/ambari/blob/89bb02c4/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java index 869441f..dc02f63 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java @@ -91,11 +91,13 @@ public class StackManagerTest { } public static StackManager createTestStackManager() throws Exception { - String stack = ClassLoader.getSystemClassLoader().getResource("stacks").getPath(); - return createTestStackManager(stack); + String stacksDir = ClassLoader.getSystemClassLoader().getResource("stacks").getPath(); + File stacksRoot = new File(stacksDir); + File resourcesRoot = stacksRoot.getParentFile(); + return createTestStackManager(resourcesRoot, stacksRoot); } - public static StackManager createTestStackManager(String stackRoot) throws Exception { + public static StackManager createTestStackManager(File resourcesRoot, File stacksRoot) throws Exception { // todo: dao , actionMetaData expectations metaInfoDao = createNiceMock(MetainfoDAO.class); stackDao = createNiceMock(StackDAO.class); @@ -122,7 +124,7 @@ public class StackManagerTest { osFamily = new OsFamily(config); AmbariManagementHelper helper = new AmbariManagementHelper(stackDao, extensionDao, linkDao); - StackManager stackManager = new StackManager(new File(stackRoot), null, null, osFamily, false, + StackManager stackManager = new StackManager(resourcesRoot, stacksRoot, null, null, osFamily, false, false, metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao, helper); verify(config, metaInfoDao, stackDao, actionMetadata); @@ -798,8 +800,8 @@ public class StackManagerTest { OsFamily osFamily = new OsFamily(config); AmbariManagementHelper helper = new AmbariManagementHelper(stackDao, extensionDao, linkDao); - StackManager stackManager = new StackManager(stackRoot, commonServices, extensions, - osFamily, false, metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao, helper); + StackManager stackManager = new StackManager(resourcesDirectory, stackRoot, commonServices, extensions, + osFamily, false, false, metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao, helper); for (StackInfo stackInfo : stackManager.getStacks()) { for (ServiceInfo serviceInfo : stackInfo.getServices()) { @@ -864,8 +866,8 @@ public class StackManagerTest { OsFamily osFamily = new OsFamily(config); AmbariManagementHelper helper = new AmbariManagementHelper(stackDao, extensionDao, linkDao); - StackManager stackManager = new StackManager(stackRoot, commonServices, extensions, osFamily, - false, metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao, helper); + StackManager stackManager = new StackManager(resourcesDirectory, stackRoot, commonServices, extensions, osFamily, + false, false, metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao, helper); String rangerUserSyncRoleCommand = Role.RANGER_USERSYNC + "-" + RoleCommand.START; String rangerAdminRoleCommand = Role.RANGER_ADMIN + "-" + RoleCommand.START; @@ -994,8 +996,8 @@ public class StackManagerTest { OsFamily osFamily = new OsFamily(config); AmbariManagementHelper helper = new AmbariManagementHelper(stackDao, extensionDao, linkDao); - StackManager stackManager = new StackManager(stackRoot, commonServices, extensions, osFamily, - false, metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao, helper); + StackManager stackManager = new StackManager(resourcesDirectory, stackRoot, commonServices, extensions, osFamily, + false, false, metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao, helper); String zookeeperServerRoleCommand = Role.ZOOKEEPER_SERVER + "-" + RoleCommand.START; String logsearchServerRoleCommand = Role.LOGSEARCH_SERVER + "-" + RoleCommand.START; http://git-wip-us.apache.org/repos/asf/ambari/blob/89bb02c4/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorUpdateHelperTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorUpdateHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorUpdateHelperTest.java index 413a1b6..7512510 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorUpdateHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorUpdateHelperTest.java @@ -36,7 +36,9 @@ import org.apache.ambari.server.orm.dao.ExtensionLinkDAO; import org.apache.ambari.server.orm.entities.ExtensionLinkEntity; import org.apache.ambari.server.orm.entities.MetainfoEntity; import org.apache.ambari.server.orm.entities.StackEntity; +import org.apache.ambari.server.stack.StackManager; import org.apache.ambari.server.stack.StackManagerFactory; +import org.apache.ambari.server.stack.StackManagerMock; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.stack.OsFamily; import org.easymock.EasyMock; @@ -71,7 +73,8 @@ public class KerberosDescriptorUpdateHelperTest extends EasyMockSupport { properties.put("mpacks-v2.staging.path","src/test/resources/mpacks-v2"); Configuration configuration = new Configuration(properties); - install(new FactoryModuleBuilder().build(StackManagerFactory.class)); + install(new FactoryModuleBuilder().implement( + StackManager.class, StackManagerMock.class).build(StackManagerFactory.class)); bind(Clusters.class).toInstance(createNiceMock(Clusters.class)); bind(DBAccessor.class).toInstance(createNiceMock(DBAccessor.class));
