Repository: stratos Updated Branches: refs/heads/stratos-4.1.x 4bc27f29c -> 7b8f70676
Move HealthStatReader, ArtifactCheckout and ArtifactCommit tasks to separate plugins in PCA Project: http://git-wip-us.apache.org/repos/asf/stratos/repo Commit: http://git-wip-us.apache.org/repos/asf/stratos/commit/68fa338f Tree: http://git-wip-us.apache.org/repos/asf/stratos/tree/68fa338f Diff: http://git-wip-us.apache.org/repos/asf/stratos/diff/68fa338f Branch: refs/heads/stratos-4.1.x Commit: 68fa338f81af2bdb7f72ae8f98f67cb04f6753c4 Parents: 4bc27f2 Author: Akila Perera <[email protected]> Authored: Tue Oct 13 03:15:21 2015 +0530 Committer: Akila Perera <[email protected]> Committed: Tue Oct 13 03:15:21 2015 +0530 ---------------------------------------------------------------------- .../cartridge.agent/cartridge.agent/agent.py | 4 +- .../cartridge.agent/cartridge.agent/config.py | 105 +++++++--- .../cartridge.agent/healthstats.py | 44 +---- .../modules/artifactmgt/git/agentgithandler.py | 194 +------------------ .../modules/event/eventhandler.py | 4 +- .../plugins/DefaultArtifactCheckout.py | 85 ++++++++ .../DefaultArtifactCheckout.yapsy-plugin | 25 +++ .../plugins/DefaultArtifactCommit.py | 129 ++++++++++++ .../plugins/DefaultArtifactCommit.yapsy-plugin | 25 +++ .../plugins/DefaultHealthStatisticsReader.py | 50 +++++ .../DefaultHealthStatisticsReader.yapsy-plugin | 25 +++ .../cartridge.agent/plugins/contracts.py | 16 +- .../cartridge.agent/publisher.py | 4 +- .../integration/tests/ADCExtensionTestCase.java | 64 +----- .../agent/integration/tests/ADCTestCase.java | 31 ++- .../extensions/py/checkout-job-handler.py | 6 +- .../extensions/py/commit-job-handler.py | 6 +- .../resources/AgentStartupTestCase/agent.conf | 4 +- .../extensions/py/checkout-job-handler.py | 34 ++++ .../py/checkout-job-handler.yapsy-plugin | 27 +++ .../extensions/py/commit-job-handler.py | 27 +++ .../py/commit-job-handler.yapsy-plugin | 27 +++ 22 files changed, 587 insertions(+), 349 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/stratos/blob/68fa338f/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/agent.py ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/agent.py b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/agent.py index 1e58dba..d37c555 100644 --- a/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/agent.py +++ b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/agent.py @@ -36,7 +36,7 @@ class CartridgeAgent(threading.Thread): def __init__(self): threading.Thread.__init__(self) - + Config.initialize_config() self.__tenant_context_initialized = False self.__log_publish_manager = None self.__terminated = False @@ -106,7 +106,7 @@ class CartridgeAgent(threading.Thread): if repo_url is None or str(repo_url).strip() == "": self.__log.info("No artifact repository found") self.__event_handler.on_instance_activated_event() - publisher.publish_instance_activated_event(Config.health_stat_plugin) + publisher.publish_instance_activated_event() else: self.__log.info( "Artifact repository found, waiting for artifact updated event to checkout artifacts: [repo_url] %s", http://git-wip-us.apache.org/repos/asf/stratos/blob/68fa338f/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/config.py ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/config.py b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/config.py index 31da21f..1f0680f 100644 --- a/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/config.py +++ b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/config.py @@ -17,13 +17,15 @@ import ConfigParser import os +import sys from yapsy.PluginManager import PluginManager from modules.util.log import LogFactory from exception import ParameterNotFoundException import constants -from plugins.contracts import ICartridgeAgentPlugin, IArtifactManagementPlugin, IHealthStatReaderPlugin +from plugins.contracts import ICartridgeAgentPlugin, IArtifactCommitPlugin, IArtifactCheckoutPlugin, \ + IHealthStatReaderPlugin class Config: @@ -31,8 +33,12 @@ class Config: Handles the configuration information of the particular Cartridge Agent """ + def __init__(self): + pass + AGENT_PLUGIN_EXT = "agent-plugin" - ARTIFACT_MGT_PLUGIN = "ArtifactManagementPlugin" + ARTIFACT_CHECKOUT_PLUGIN = "ArtifactCheckoutPlugin" + ARTIFACT_COMMIT_PLUGIN = "ArtifactCommitPlugin" CARTRIDGE_AGENT_PLUGIN = "CartridgeAgentPlugin" HEALTH_STAT_PLUGIN = "HealthStatReaderPlugin" @@ -45,7 +51,8 @@ class Config: plugins = {} """ :type dict{str: [PluginInfo]} : """ - artifact_mgt_plugins = [] + artifact_checkout_plugin = None + artifact_commit_plugin = None health_stat_plugin = None extension_executor = None @@ -225,10 +232,6 @@ class Config: :return: Value of the property :exception: ParameterNotFoundException if the provided property cannot be found """ - - if Config.properties is None or Config.payload_params == {}: - Config.initialize_config() - if Config.properties.has_option("agent", property_key): temp_str = Config.properties.get("agent", property_key) Config.log.debug("Reading property: %s = %s", property_key, temp_str) @@ -272,15 +275,15 @@ class Config: Config.tenant_id = Config.read_property(constants.TENANT_ID) try: - Config.is_clustered = Config.read_property(constants.CLUSTERING) + Config.is_clustered = Config.read_property(constants.CLUSTERING, False) except ParameterNotFoundException: Config.is_clustered = False try: - Config.is_commits_enabled = Config.read_property(constants.COMMIT_ENABLED) + Config.is_commits_enabled = Config.read_property(constants.COMMIT_ENABLED, False) except ParameterNotFoundException: try: - Config.is_commits_enabled = Config.read_property(constants.AUTO_COMMIT) + Config.is_commits_enabled = Config.read_property(constants.AUTO_COMMIT, False) except ParameterNotFoundException: Config.is_commits_enabled = False @@ -353,7 +356,7 @@ class Config: Config.log.debug("log_file_paths: %s" % Config.log_file_paths) Config.log.info("Initializing plugins") - Config.plugins, Config.artifact_mgt_plugins, Config.health_stat_plugin = Config.initialize_plugins() + Config.initialize_plugins() Config.extension_executor = Config.initialize_extensions() @staticmethod @@ -367,7 +370,8 @@ class Config: # TODO: change plugin descriptor ext, plugin_manager.setPluginInfoExtension(AGENT_PLUGIN_EXT) plugins_dir = Config.read_property(constants.PLUGINS_DIR) category_filter = {Config.CARTRIDGE_AGENT_PLUGIN: ICartridgeAgentPlugin, - Config.ARTIFACT_MGT_PLUGIN: IArtifactManagementPlugin, + Config.ARTIFACT_CHECKOUT_PLUGIN: IArtifactCheckoutPlugin, + Config.ARTIFACT_COMMIT_PLUGIN: IArtifactCommitPlugin, Config.HEALTH_STAT_PLUGIN: IHealthStatReaderPlugin} plugin_manager = Config.create_plugin_manager(category_filter, plugins_dir) @@ -387,34 +391,79 @@ class Config: grouped_ca_plugins[mapped_event] = [] grouped_ca_plugins[mapped_event].append(plugin_info) + Config.plugins = grouped_ca_plugins # activate artifact management plugins - artifact_mgt_plugins = plugin_manager.getPluginsOfCategory(Config.ARTIFACT_MGT_PLUGIN) - for plugin_info in artifact_mgt_plugins: - # TODO: Fix this to only load the first plugin - Config.log.debug("Found artifact management plugin [%s] at [%s]" % (plugin_info.name, plugin_info.path)) - plugin_manager.activatePluginByName(plugin_info.name) - Config.log.info("Activated artifact management plugin [%s]" % plugin_info.name) + artifact_checkout_plugins = plugin_manager.getPluginsOfCategory(Config.ARTIFACT_CHECKOUT_PLUGIN) + for plugin_info in artifact_checkout_plugins: + Config.log.debug("Found artifact checkout plugin [%s] at [%s]" % (plugin_info.name, plugin_info.path)) + # if multiple artifact management plugins are registered, halt agent execution. This is to avoid any + # undesired outcome due to errors made in deployment + if Config.is_checkout_enabled: + if len(artifact_checkout_plugins) == 0: + Config.log.exception( + "No plugins registered for artifact checkout extension. Stratos agent failed to start") + sys.exit(1) + elif len(artifact_checkout_plugins) == 1: + plugin_info = artifact_checkout_plugins[0] + Config.log.debug("Found artifact checkout plugin [%s] at [%s]" % + (plugin_info.name, plugin_info.path)) + plugin_manager.activatePluginByName(plugin_info.name) + Config.log.info("Activated artifact checkout plugin [%s]" % plugin_info.name) + Config.artifact_checkout_plugin = plugin_info + elif len(artifact_checkout_plugins) > 1: + Config.log.exception( + "Multiple plugins registered for artifact checkout. Stratos agent failed to start.") + sys.exit(1) + + artifact_commit_plugins = plugin_manager.getPluginsOfCategory(Config.ARTIFACT_COMMIT_PLUGIN) + for plugin_info in artifact_commit_plugins: + Config.log.debug("Found artifact commit plugin [%s] at [%s]" % (plugin_info.name, plugin_info.path)) + if Config.is_commits_enabled: + if len(artifact_commit_plugins) == 0: + Config.log.exception( + "No plugins registered for artifact commit extension. Stratos agent failed to start") + sys.exit(1) + elif len(artifact_commit_plugins) == 1: + plugin_info = artifact_commit_plugins[0] + Config.log.debug("Found artifact commit plugin [%s] at [%s]" % + (plugin_info.name, plugin_info.path)) + plugin_manager.activatePluginByName(plugin_info.name) + Config.log.info("Activated artifact commit plugin [%s]" % plugin_info.name) + Config.artifact_commit_plugin = plugin_info + elif len(artifact_commit_plugins) > 1: + Config.log.exception( + "Multiple plugins registered for artifact checkout. Stratos agent failed to start.") + sys.exit(1) health_stat_plugins = plugin_manager.getPluginsOfCategory(Config.HEALTH_STAT_PLUGIN) - health_stat_plugin = None - - # If there are any health stat reader plugins, load the first one and ignore the rest - if len(health_stat_plugins) > 0: + for plugin_info in health_stat_plugins: + Config.log.debug("Found health stats reader plugin [%s] at [%s]" % (plugin_info.name, plugin_info.path)) + # If multiple health stat reader plugins are registered, halt agent execution. This is to avoid any + # undesired outcome due to errors made in deployment + if len(health_stat_plugins) == 0: + Config.log.exception( + "No plugins registered for health statistics reader. Stratos agent failed to start.") + sys.exit(1) + elif len(health_stat_plugins) == 1: plugin_info = health_stat_plugins[0] Config.log.debug("Found health statistics reader plugin [%s] at [%s]" % (plugin_info.name, plugin_info.path)) plugin_manager.activatePluginByName(plugin_info.name) Config.log.info("Activated health statistics reader plugin [%s]" % plugin_info.name) - health_stat_plugin = plugin_info - - return grouped_ca_plugins, artifact_mgt_plugins, health_stat_plugin + Config.health_stat_plugin = plugin_info + elif len(health_stat_plugins) > 1: + Config.log.exception( + "Multiple plugins registered for health statistics reader. Stratos agent failed to start.") + sys.exit(1) except ParameterNotFoundException as e: Config.log.exception("Could not load plugins. Plugins directory not set: %s" % e) - return None, None + Config.log.error("Stratos agent failed to start") + sys.exit(1) except Exception as e: - Config.log.exception("Error while loading plugin: %s" % e) - return None, None + Config.log.exception("Error while loading plugins: %s" % e) + Config.log.error("Stratos agent failed to start") + sys.exit(1) @staticmethod def initialize_extensions(): http://git-wip-us.apache.org/repos/asf/stratos/blob/68fa338f/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/healthstats.py ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/healthstats.py b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/healthstats.py index 69c2008..877bde0 100644 --- a/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/healthstats.py +++ b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/healthstats.py @@ -16,10 +16,6 @@ # under the License. from threading import Thread -import multiprocessing - -import psutil - from modules.databridge.agent import * from config import Config from modules.util import cartridgeagentutils @@ -37,7 +33,7 @@ class HealthStatisticsPublisherManager(Thread): STREAM_NICKNAME = "agent health stats" STREAM_DESCRIPTION = "agent health stats" - def __init__(self, publish_interval, health_stat_plugin): + def __init__(self, publish_interval): """ Initializes a new HealthStatisticsPublisherManager with a given number of seconds as the interval :param int publish_interval: Number of seconds as the interval @@ -50,8 +46,9 @@ class HealthStatisticsPublisherManager(Thread): self.terminated = False self.publisher = HealthStatisticsPublisher() """:type : HealthStatisticsPublisher""" - # If there are no health stat reader plugins, create the default reader instance - self.stats_reader = health_stat_plugin if health_stat_plugin is not None else DefaultHealthStatisticsReader() + + """:type : IHealthStatReaderPlugin""" + self.stats_reader = Config.health_stat_plugin def run(self): while not self.terminated: @@ -59,14 +56,15 @@ class HealthStatisticsPublisherManager(Thread): try: ca_health_stat = CartridgeHealthStatistics() - cartridge_stats = self.stats_reader.stat_cartridge_health(ca_health_stat) + cartridge_stats = self.stats_reader.plugin_object.stat_cartridge_health(ca_health_stat) self.log.debug("Publishing memory consumption: %r" % cartridge_stats.memory_usage) self.publisher.publish_memory_usage(cartridge_stats.memory_usage) self.log.debug("Publishing load average: %r" % cartridge_stats.load_avg) self.publisher.publish_load_average(cartridge_stats.load_avg) except ThriftReceiverOfflineException: - self.log.exception("Couldn't publish health statistics to CEP. Thrift Receiver offline. Reconnecting...") + self.log.exception( + "Couldn't publish health statistics to CEP. Thrift Receiver offline. Reconnecting...") self.publisher = HealthStatisticsPublisher() self.publisher.disconnect_publisher() @@ -118,7 +116,6 @@ class HealthStatisticsPublisher: if not cep_active: raise CEPPublisherException("CEP server not active. Health statistics publishing aborted.") - self.stream_definition = HealthStatisticsPublisher.create_stream_definition() HealthStatisticsPublisher.log.debug("Stream definition created: %r" % str(self.stream_definition)) @@ -212,33 +209,6 @@ class HealthStatisticsPublisher: for publisher in self.publishers: publisher.disconnect() -class DefaultHealthStatisticsReader: - """ - Default implementation for the health statistics reader. If no Health Statistics Reader plugins are provided, - this will be used to read health stats from the instance. - """ - - def __init__(self): - self.log = LogFactory().get_log(__name__) - - def stat_cartridge_health(self, ca_health_stat): - ca_health_stat.memory_usage = DefaultHealthStatisticsReader.__read_mem_usage() - ca_health_stat.load_avg = DefaultHealthStatisticsReader.__read_load_avg() - - self.log.debug("Memory read: %r, CPU read: %r" % (ca_health_stat.memory_usage, ca_health_stat.load_avg)) - return ca_health_stat - - @staticmethod - def __read_mem_usage(): - return psutil.virtual_memory().percent - - @staticmethod - def __read_load_avg(): - (one, five, fifteen) = os.getloadavg() - cores = multiprocessing.cpu_count() - - return (one/cores) * 100 - class CartridgeHealthStatistics: """ http://git-wip-us.apache.org/repos/asf/stratos/blob/68fa338f/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/modules/artifactmgt/git/agentgithandler.py ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/modules/artifactmgt/git/agentgithandler.py b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/modules/artifactmgt/git/agentgithandler.py index 0e79091..59c25d9 100644 --- a/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/modules/artifactmgt/git/agentgithandler.py +++ b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/modules/artifactmgt/git/agentgithandler.py @@ -47,77 +47,6 @@ class AgentGitHandler: # (tenant_id => GitRepository) @staticmethod - def run_checkout_job(repo_info): - """ - Checks out the code from the remote repository. - If local repository path is empty, a clone operation is done. - If there is a cloned repository already on the local repository path, a pull operation - will be performed. - If there are artifacts not in the repository already on the local repository path, - they will be added to a git repository, the remote url added as origin, and then - a pull operation will be performed. - - :param Repository repo_info: The repository information object - :return: A tuple containing whether it was an initial clone or not, and if the repo was updated on - subsequent calls or not - :rtype: tuple(bool, bool) - """ - plugins_for_event = Config.plugins.get(constants.ARTIFACT_CHECKOUT_JOB) - if plugins_for_event is not None and len(plugins_for_event) > 0: - if len(plugins_for_event) > 1: - for plugin_info in plugins_for_event: - AgentGitHandler.log.debug("Registered plugin name for commit job: %s" % plugin_info.name) - AgentGitHandler.log.error("More than one plugin registered for checkout job. Aborting...") - return - try: - plugin_values = {"REPO_INFO": repo_info} - plugins_for_event[0].plugin_object.run_plugin(plugin_values) - return - except Exception as e: - AgentGitHandler.log.exception("Error while executing checkout job extension: %s" % e) - - git_repo = AgentGitHandler.create_git_repo(repo_info) - if AgentGitHandler.get_repo(repo_info.tenant_id) is not None: - # has been previously cloned, this is not the subscription run - if AgentGitHandler.is_valid_git_repository(git_repo): - AgentGitHandler.log.debug("Executing git pull: [tenant-id] %s [repo-url] %s", - git_repo.tenant_id, git_repo.repo_url) - try: - updated = AgentGitHandler.pull(git_repo) - AgentGitHandler.log.debug("Git pull executed: [tenant-id] %s [repo-url] %s", - git_repo.tenant_id, git_repo.repo_url) - except GitRepositorySynchronizationException as e: - AgentGitHandler.log.exception("Git pull operation failed: %s" % e) - - else: - # not a valid repository, might've been corrupted. do a re-clone - AgentGitHandler.log.debug("Local repository is not valid. Doing a re-clone to purify.") - git_repo.cloned = False - AgentGitHandler.log.debug("Executing git clone: [tenant-id] %s [repo-url] %s", - git_repo.tenant_id, git_repo.repo_url) - git_repo = AgentGitHandler.clone(git_repo) - AgentGitHandler.add_repo(git_repo) - AgentGitHandler.log.debug("Git clone executed: [tenant-id] %s [repo-url] %s", - git_repo.tenant_id, git_repo.repo_url) - else: - # subscribing run.. need to clone - AgentGitHandler.log.info("Cloning artifacts from %s for the first time to %s", - git_repo.repo_url, git_repo.local_repo_path) - AgentGitHandler.log.info("Executing git clone: [tenant-id] %s [repo-url] %s, [repo path] %s", - git_repo.tenant_id, git_repo.repo_url, git_repo.local_repo_path) - try: - git_repo = AgentGitHandler.clone(git_repo) - AgentGitHandler.add_repo(git_repo) - AgentGitHandler.log.debug("Git clone executed: [tenant-id] %s [repo-url] %s", - git_repo.tenant_id, git_repo.repo_url) - except GitRepositorySynchronizationException as e: - AgentGitHandler.log.exception("Git clone operation failed: %s" % e) - # If first git clone is failed, execute retry_clone operation - AgentGitHandler.log.info("Retrying git clone operation...") - AgentGitHandler.retry_clone(git_repo) - AgentGitHandler.add_repo(git_repo) - - @staticmethod def sync_initial_local_artifacts(git_repo): # init git repo AgentGitHandler.init(git_repo.local_repo_path) @@ -325,115 +254,6 @@ class AgentGitHandler: return repo_info.repo_url @staticmethod - def run_commit_job(repo_info): - """ - Commits and pushes new artifacts to the remote repository - :param repo_info: - :return: - """ - plugins_for_event = Config.plugins.get(constants.ARTIFACT_COMMIT_JOB) - if plugins_for_event is not None and len(plugins_for_event) > 0: - if len(plugins_for_event) > 1: - for plugin_info in plugins_for_event: - AgentGitHandler.log.debug("Registered plugin name for commit job: %s" % plugin_info.name) - AgentGitHandler.log.error("More than one plugin registered for commit job. Aborting...") - return - try: - plugins_for_event[0].plugin_object.run_plugin({}) - return - except Exception as e: - AgentGitHandler.log.exception("Error while executing commit job extension: %s " % e) - - git_repo = AgentGitHandler.get_repo(repo_info.tenant_id) - if git_repo is None: - # not cloned yet - AgentGitHandler.log.error("Not a valid repository to push from. Aborting Git push...") - - # Get initial HEAD so in case if push fails it can be reverted to this hash - # This way, commit and push becomes an single operation. No intermediate state will be left behind. - (init_head, init_errors) = AgentGitHandler.execute_git_command(["rev-parse", "HEAD"], git_repo.local_repo_path) - - # remove trailing new line character, if any - init_head = init_head.rstrip() - - # stage all untracked files - if AgentGitHandler.stage_all(git_repo.local_repo_path): - AgentGitHandler.log.debug("Git staged untracked artifacts successfully") - else: - AgentGitHandler.log.error("Git could not stage untracked artifacts") - - # check for changes in working directory - modified = AgentGitHandler.has_modified_files(git_repo.local_repo_path) - - AgentGitHandler.log.debug("[Git] Modified: %s" % str(modified)) - if not modified: - AgentGitHandler.log.debug("No changes detected in the local repository for tenant %s" % git_repo.tenant_id) - return - - # commit to local repository - commit_message = "tenant [%s]'s artifacts committed to local repo at %s" \ - % (git_repo.tenant_id, git_repo.local_repo_path) - # TODO: set configuratble names, check if already configured - commit_name = git_repo.tenant_id - commit_email = "[email protected]" - # git config - AgentGitHandler.execute_git_command(["config", "user.email", commit_email], git_repo.local_repo_path) - AgentGitHandler.execute_git_command(["config", "user.name", commit_name], git_repo.local_repo_path) - - # commit - (output, errors) = AgentGitHandler.execute_git_command(["commit", "-m", commit_message], - git_repo.local_repo_path) - if errors.strip() == "": - commit_hash = AgentGitHandler.find_between(output, "[master", "]").strip() - AgentGitHandler.log.debug("Committed artifacts for tenant: %s : %s " % (git_repo.tenant_id, commit_hash)) - else: - AgentGitHandler.log.error("Committing artifacts to local repository failed for tenant: %s, Cause: %s" - % (git_repo.tenant_id, errors)) - # revert to initial commit hash - AgentGitHandler.execute_git_command(["reset", "--hard", init_head], git_repo.local_repo_path) - return - - repo = Repo(git_repo.local_repo_path) - # pull and rebase before pushing to remote repo - AgentGitHandler.execute_git_command(["pull", "--rebase", "origin", "master"], git_repo.local_repo_path) - if repo.is_dirty(): - AgentGitHandler.log.error("Git pull operation in commit job left the repository in dirty state") - AgentGitHandler.log.error( - "Git pull rebase operation on remote %s for tenant %s failed" % (git_repo.repo_url, git_repo.tenant_id)) - - AgentGitHandler.log.warn("The working directory will be reset to the last known good commit") - # revert to the initial commit - AgentGitHandler.execute_git_command(["reset", "--hard", init_head], git_repo.local_repo_path) - return - else: - # push to remote - try: - push_info_list = repo.remotes.origin.push() - if (len(push_info_list)) == 0: - AgentGitHandler.log.error("Failed to push artifacts to remote repo for tenant: %s remote: %s" % - (git_repo.tenant_id, git_repo.repo_url)) - # revert to the initial commit - AgentGitHandler.execute_git_command(["reset", "--hard", init_head], git_repo.local_repo_path) - return - - for push_info in push_info_list: - AgentGitHandler.log.debug("Push info summary: %s" % push_info.summary) - if push_info.flags & PushInfo.ERROR == PushInfo.ERROR: - AgentGitHandler.log.error("Failed to push artifacts to remote repo for tenant: %s remote: %s" % - (git_repo.tenant_id, git_repo.repo_url)) - # revert to the initial commit - AgentGitHandler.execute_git_command(["reset", "--hard", init_head], git_repo.local_repo_path) - return - AgentGitHandler.log.debug( - "Successfully pushed artifacts for tenant: %s remote: %s" % (git_repo.tenant_id, git_repo.repo_url)) - except Exception as e: - AgentGitHandler.log.error( - "Failed to push artifacts to remote repo for tenant: %s remote: %s exception: %s" % - (git_repo.tenant_id, git_repo.repo_url, e)) - # revert to the initial commit - AgentGitHandler.execute_git_command(["reset", "--hard", init_head], git_repo.local_repo_path) - - @staticmethod def has_modified_files(repo_path): (output, errors) = AgentGitHandler.execute_git_command(["status"], repo_path=repo_path) if "nothing to commit" in output: @@ -524,30 +344,26 @@ class ArtifactUpdateTask(AbstractAsyncScheduledTask): self.repo_info = repo_info self.auto_checkout = auto_checkout self.auto_commit = auto_commit - self.invocation_count = 0 def execute_task(self): - self.invocation_count += 1 - # DO NOT change this order. The commit job should run first here. # This is because if the cloned location contain any un-tracked files then # those files should be committed and pushed first if self.auto_commit: try: - self.log.debug("Running commit job # %s" % self.invocation_count) - AgentGitHandler.run_commit_job(self.repo_info) + self.log.debug("Running commit job...") + Config.artifact_commit_plugin.plugin_object.commit(self.repo_info) except GitRepositorySynchronizationException as e: self.log.exception("Auto commit failed: %s" % e) if self.auto_checkout: try: - self.log.debug("Running checkout job # %s" % self.invocation_count) - AgentGitHandler.run_checkout_job(self.repo_info) - # TODO: move this to updated scheduler extension + self.log.debug("Running checkout job...") + Config.artifact_checkout_plugin.plugin_object.checkout(self.repo_info) except GitRepositorySynchronizationException as e: self.log.exception("Auto checkout task failed: %s" % e) - self.log.debug("ArtifactUpdateTask completed # %s" % self.invocation_count) + self.log.debug("ArtifactUpdateTask end of iteration.") class GitRepository: http://git-wip-us.apache.org/repos/asf/stratos/blob/68fa338f/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/modules/event/eventhandler.py ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/modules/event/eventhandler.py b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/modules/event/eventhandler.py index 259d30d..7fbdf5f 100644 --- a/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/modules/event/eventhandler.py +++ b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/modules/event/eventhandler.py @@ -109,7 +109,7 @@ class EventHandler: self.__log.info("Executing checkout job on artifact updated event...") try: - AgentGitHandler.run_checkout_job(repo_info) + Config.artifact_checkout_plugin.plugin_object.checkout(repo_info) except GitRepositorySynchronizationException as e: self.__log.exception( "Checkout job on artifact updated event failed for tenant: %s %s" % (repo_info.tenant_id, e)) @@ -129,7 +129,7 @@ class EventHandler: if existing_git_repo is None: # publish instance activated event for single tenant subscription - publisher.publish_instance_activated_event(Config.health_stat_plugin) + publisher.publish_instance_activated_event() update_artifacts = Config.read_property(constants.ENABLE_ARTIFACT_UPDATE, True) auto_commit = Config.is_commits_enabled http://git-wip-us.apache.org/repos/asf/stratos/blob/68fa338f/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/plugins/DefaultArtifactCheckout.py ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/plugins/DefaultArtifactCheckout.py b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/plugins/DefaultArtifactCheckout.py new file mode 100644 index 0000000..53cadea --- /dev/null +++ b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/plugins/DefaultArtifactCheckout.py @@ -0,0 +1,85 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from plugins.contracts import IArtifactCheckoutPlugin +from modules.util.log import LogFactory +from modules.artifactmgt.git.agentgithandler import AgentGitHandler +from config import Config +import constants +from exception import * + + +class DefaultArtifactCheckout(IArtifactCheckoutPlugin): + """ + Default implementation for the artifact checkout handling + """ + + def __init__(self): + super(DefaultArtifactCheckout, self).__init__() + self.log = LogFactory().get_log(__name__) + + def checkout(self, repo_info): + """ + Checks out the code from the remote repository. + If local repository path is empty, a clone operation is done. + If there is a cloned repository already on the local repository path, a pull operation + will be performed. + If there are artifacts not in the repository already on the local repository path, + they will be added to a git repository, the remote url added as origin, and then + a pull operation will be performed. + + :param Repository repo_info: The repository information object + :return: A tuple containing whether it was an initial clone or not, and if the repo was updated on + subsequent calls or not + :rtype: tuple(bool, bool) + """ + git_repo = AgentGitHandler.create_git_repo(repo_info) + if AgentGitHandler.get_repo(repo_info.tenant_id) is not None: + # has been previously cloned, this is not the subscription run + if AgentGitHandler.is_valid_git_repository(git_repo): + AgentGitHandler.log.debug("Executing git pull: [tenant-id] %s [repo-url] %s", + git_repo.tenant_id, git_repo.repo_url) + updated = AgentGitHandler.pull(git_repo) + AgentGitHandler.log.debug("Git pull executed: [tenant-id] %s [repo-url] %s", + git_repo.tenant_id, git_repo.repo_url) + else: + # not a valid repository, might've been corrupted. do a re-clone + AgentGitHandler.log.debug("Local repository is not valid. Doing a re-clone to purify.") + git_repo.cloned = False + AgentGitHandler.log.debug("Executing git clone: [tenant-id] %s [repo-url] %s", + git_repo.tenant_id, git_repo.repo_url) + git_repo = AgentGitHandler.clone(git_repo) + AgentGitHandler.add_repo(git_repo) + AgentGitHandler.log.debug("Git clone executed: [tenant-id] %s [repo-url] %s", + git_repo.tenant_id, git_repo.repo_url) + else: + # subscribing run.. need to clone + AgentGitHandler.log.info("Cloning artifacts from %s for the first time to %s", + git_repo.repo_url, git_repo.local_repo_path) + AgentGitHandler.log.info("Executing git clone: [tenant-id] %s [repo-url] %s, [repo path] %s", + git_repo.tenant_id, git_repo.repo_url, git_repo.local_repo_path) + try: + git_repo = AgentGitHandler.clone(git_repo) + AgentGitHandler.add_repo(git_repo) + AgentGitHandler.log.debug("Git clone executed: [tenant-id] %s [repo-url] %s", + git_repo.tenant_id, git_repo.repo_url) + except GitRepositorySynchronizationException as e: + AgentGitHandler.log.exception("Git clone operation failed: %s" % e) + # If first git clone is failed, execute retry_clone operation + AgentGitHandler.log.info("Retrying git clone operation...") + AgentGitHandler.retry_clone(git_repo) + AgentGitHandler.add_repo(git_repo) http://git-wip-us.apache.org/repos/asf/stratos/blob/68fa338f/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/plugins/DefaultArtifactCheckout.yapsy-plugin ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/plugins/DefaultArtifactCheckout.yapsy-plugin b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/plugins/DefaultArtifactCheckout.yapsy-plugin new file mode 100644 index 0000000..871baf4 --- /dev/null +++ b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/plugins/DefaultArtifactCheckout.yapsy-plugin @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[Core] +Name = DefaultArtifactCheckout +Module = DefaultArtifactCheckout + +[Documentation] +Author = Op1 +Version = 0.1 +Website = stratos.apache.org http://git-wip-us.apache.org/repos/asf/stratos/blob/68fa338f/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/plugins/DefaultArtifactCommit.py ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/plugins/DefaultArtifactCommit.py b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/plugins/DefaultArtifactCommit.py new file mode 100644 index 0000000..c9f5dd0 --- /dev/null +++ b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/plugins/DefaultArtifactCommit.py @@ -0,0 +1,129 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from plugins.contracts import IArtifactCommitPlugin +from modules.util.log import LogFactory +from modules.artifactmgt.git.agentgithandler import AgentGitHandler +from config import Config +import constants +from exception import * +from git import * + + +class DefaultArtifactCommit(IArtifactCommitPlugin): + """ + Default implementation for the artifact checkout handling + """ + + def __init__(self): + super(DefaultArtifactCommit, self).__init__() + self.log = LogFactory().get_log(__name__) + + def commit(self, repo_info): + """ + Commits and pushes new artifacts to the remote repository + :param repo_info: + :return: + """ + git_repo = AgentGitHandler.get_repo(repo_info.tenant_id) + if git_repo is None: + # not cloned yet + AgentGitHandler.log.error("Not a valid repository to push from. Aborting Git push...") + + # Get initial HEAD so in case if push fails it can be reverted to this hash + # This way, commit and push becomes an single operation. No intermediate state will be left behind. + (init_head, init_errors) = AgentGitHandler.execute_git_command(["rev-parse", "HEAD"], git_repo.local_repo_path) + + # remove trailing new line character, if any + init_head = init_head.rstrip() + + # stage all untracked files + if AgentGitHandler.stage_all(git_repo.local_repo_path): + AgentGitHandler.log.debug("Git staged untracked artifacts successfully") + else: + AgentGitHandler.log.error("Git could not stage untracked artifacts") + + # check for changes in working directory + modified = AgentGitHandler.has_modified_files(git_repo.local_repo_path) + + AgentGitHandler.log.debug("[Git] Modified: %s" % str(modified)) + if not modified: + AgentGitHandler.log.debug("No changes detected in the local repository for tenant %s" % git_repo.tenant_id) + return + + # commit to local repository + commit_message = "tenant [%s]'s artifacts committed to local repo at %s" \ + % (git_repo.tenant_id, git_repo.local_repo_path) + # TODO: set configuratble names, check if already configured + commit_name = git_repo.tenant_id + commit_email = "[email protected]" + # git config + AgentGitHandler.execute_git_command(["config", "user.email", commit_email], git_repo.local_repo_path) + AgentGitHandler.execute_git_command(["config", "user.name", commit_name], git_repo.local_repo_path) + + # commit + (output, errors) = AgentGitHandler.execute_git_command(["commit", "-m", commit_message], + git_repo.local_repo_path) + if errors.strip() == "": + commit_hash = AgentGitHandler.find_between(output, "[master", "]").strip() + AgentGitHandler.log.debug("Committed artifacts for tenant: %s : %s " % (git_repo.tenant_id, commit_hash)) + else: + AgentGitHandler.log.error("Committing artifacts to local repository failed for tenant: %s, Cause: %s" + % (git_repo.tenant_id, errors)) + # revert to initial commit hash + AgentGitHandler.execute_git_command(["reset", "--hard", init_head], git_repo.local_repo_path) + return + + repo = Repo(git_repo.local_repo_path) + # pull and rebase before pushing to remote repo + AgentGitHandler.execute_git_command(["pull", "--rebase", "origin", "master"], git_repo.local_repo_path) + if repo.is_dirty(): + AgentGitHandler.log.error("Git pull operation in commit job left the repository in dirty state") + AgentGitHandler.log.error( + "Git pull rebase operation on remote %s for tenant %s failed" % (git_repo.repo_url, git_repo.tenant_id)) + + AgentGitHandler.log.warn("The working directory will be reset to the last known good commit") + # revert to the initial commit + AgentGitHandler.execute_git_command(["reset", "--hard", init_head], git_repo.local_repo_path) + return + else: + # push to remote + try: + push_info_list = repo.remotes.origin.push() + if (len(push_info_list)) == 0: + AgentGitHandler.log.error("Failed to push artifacts to remote repo for tenant: %s remote: %s" % + (git_repo.tenant_id, git_repo.repo_url)) + # revert to the initial commit + AgentGitHandler.execute_git_command(["reset", "--hard", init_head], git_repo.local_repo_path) + return + + for push_info in push_info_list: + AgentGitHandler.log.debug("Push info summary: %s" % push_info.summary) + if push_info.flags & PushInfo.ERROR == PushInfo.ERROR: + AgentGitHandler.log.error("Failed to push artifacts to remote repo for tenant: %s remote: %s" % + (git_repo.tenant_id, git_repo.repo_url)) + # revert to the initial commit + AgentGitHandler.execute_git_command(["reset", "--hard", init_head], git_repo.local_repo_path) + return + AgentGitHandler.log.debug( + "Successfully pushed artifacts for tenant: %s remote: %s" % (git_repo.tenant_id, git_repo.repo_url)) + except Exception as e: + AgentGitHandler.log.error( + "Failed to push artifacts to remote repo for tenant: %s remote: %s exception: %s" % + (git_repo.tenant_id, git_repo.repo_url, e)) + # revert to the initial commit + AgentGitHandler.execute_git_command(["reset", "--hard", init_head], git_repo.local_repo_path) http://git-wip-us.apache.org/repos/asf/stratos/blob/68fa338f/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/plugins/DefaultArtifactCommit.yapsy-plugin ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/plugins/DefaultArtifactCommit.yapsy-plugin b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/plugins/DefaultArtifactCommit.yapsy-plugin new file mode 100644 index 0000000..ea602cd --- /dev/null +++ b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/plugins/DefaultArtifactCommit.yapsy-plugin @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[Core] +Name = DefaultArtifactCommit +Module = DefaultArtifactCommit + +[Documentation] +Author = Op1 +Version = 0.1 +Website = stratos.apache.org http://git-wip-us.apache.org/repos/asf/stratos/blob/68fa338f/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/plugins/DefaultHealthStatisticsReader.py ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/plugins/DefaultHealthStatisticsReader.py b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/plugins/DefaultHealthStatisticsReader.py new file mode 100644 index 0000000..a202499 --- /dev/null +++ b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/plugins/DefaultHealthStatisticsReader.py @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import os + +from plugins.contracts import IHealthStatReaderPlugin +from modules.util.log import LogFactory +import multiprocessing +import psutil + + +class DefaultHealthStatisticsReader(IHealthStatReaderPlugin): + """ + Default implementation for the health statistics reader + """ + + def __init__(self): + super(DefaultHealthStatisticsReader, self).__init__() + self.log = LogFactory().get_log(__name__) + + def stat_cartridge_health(self, ca_health_stat): + ca_health_stat.memory_usage = DefaultHealthStatisticsReader.__read_mem_usage() + ca_health_stat.load_avg = DefaultHealthStatisticsReader.__read_load_avg() + + self.log.debug("Memory read: %r, CPU read: %r" % (ca_health_stat.memory_usage, ca_health_stat.load_avg)) + return ca_health_stat + + @staticmethod + def __read_mem_usage(): + return psutil.virtual_memory().percent + + @staticmethod + def __read_load_avg(): + (one, five, fifteen) = os.getloadavg() + cores = multiprocessing.cpu_count() + + return (one / cores) * 100 http://git-wip-us.apache.org/repos/asf/stratos/blob/68fa338f/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/plugins/DefaultHealthStatisticsReader.yapsy-plugin ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/plugins/DefaultHealthStatisticsReader.yapsy-plugin b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/plugins/DefaultHealthStatisticsReader.yapsy-plugin new file mode 100644 index 0000000..3775666 --- /dev/null +++ b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/plugins/DefaultHealthStatisticsReader.yapsy-plugin @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[Core] +Name = DefaultHealthStatisticsReader +Module = DefaultHealthStatisticsReader + +[Documentation] +Author = Op1 +Version = 0.1 +Website = stratos.apache.org http://git-wip-us.apache.org/repos/asf/stratos/blob/68fa338f/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/plugins/contracts.py ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/plugins/contracts.py b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/plugins/contracts.py index 39baddc..93041da 100644 --- a/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/plugins/contracts.py +++ b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/plugins/contracts.py @@ -27,15 +27,23 @@ class ICartridgeAgentPlugin(IPlugin): raise NotImplementedError -class IArtifactManagementPlugin(IPlugin): +class IArtifactCheckoutPlugin(IPlugin): """ To implement an artifact management plugin to manage artifact distribution using a custom version control tool + It should implement checkout method which will pull artifacts from a remote repository """ - def checkout(self): + def checkout(self, repo_info): raise NotImplementedError - def push(self): + +class IArtifactCommitPlugin(IPlugin): + """ + To implement an artifact management plugin to manage artifact distribution using a custom version control tool + It should implement commit method which will push artifacts to a remote repository + """ + + def commit(self, repo_info): raise NotImplementedError @@ -45,4 +53,4 @@ class IHealthStatReaderPlugin(IPlugin): """ def stat_cartridge_health(self, health_stat): - raise NotImplementedError \ No newline at end of file + raise NotImplementedError http://git-wip-us.apache.org/repos/asf/stratos/blob/68fa338f/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/publisher.py ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/publisher.py b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/publisher.py index 6a015f4..f0ca275 100644 --- a/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/publisher.py +++ b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/publisher.py @@ -67,7 +67,7 @@ def publish_instance_started_event(): log.warn("Instance already started") -def publish_instance_activated_event(health_stat_plugin): +def publish_instance_activated_event(): global activated, log if not activated: # Wait for all ports to be active @@ -115,7 +115,7 @@ def publish_instance_activated_event(health_stat_plugin): interval = interval_default else: interval = interval_default - health_stats_publisher = healthstats.HealthStatisticsPublisherManager(interval, health_stat_plugin) + health_stats_publisher = healthstats.HealthStatisticsPublisherManager(interval) log.info("Starting Health statistics publisher with interval %r" % interval) health_stats_publisher.start() else: http://git-wip-us.apache.org/repos/asf/stratos/blob/68fa338f/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/ADCExtensionTestCase.java ---------------------------------------------------------------------- diff --git a/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/ADCExtensionTestCase.java b/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/ADCExtensionTestCase.java index 5f321fe..2398099 100644 --- a/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/ADCExtensionTestCase.java +++ b/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/ADCExtensionTestCase.java @@ -52,10 +52,7 @@ public class ADCExtensionTestCase extends PythonAgentIntegrationTest { private static final String PARTITION_ID = "partition-1"; private static final String TENANT_ID = "-1234"; private static final String SERVICE_NAME = "tomcat"; - private boolean hasADCExtensionTestCompleted = false; - private boolean hasCheckoutExtensionStarted = false; - private boolean hasCommitExtensionStarted = false; @BeforeMethod(alwaysRun = true) public void setupADCExtensionTest() throws Exception { @@ -82,7 +79,6 @@ public class ADCExtensionTestCase extends PythonAgentIntegrationTest { @Test(timeOut = ADC_TIMEOUT) public void testADC() throws Exception { startCommunicatorThread(); - assertAgentActivation(); Thread adcExtensionTestThread = new Thread(new Runnable() { @Override public void run() { @@ -97,11 +93,10 @@ public class ADCExtensionTestCase extends PythonAgentIntegrationTest { List<String> newLines = getNewLines(outputLines, outputStream.toString()); if (newLines.size() > 0) { for (String line : newLines) { - if (line.contains("Running extension for checkout job")) { - hasCheckoutExtensionStarted = true; - } - if (line.contains("Running extension for commit job")) { - hasCommitExtensionStarted = true; + if (line.contains( + "Multiple plugins registered for artifact checkout. Stratos agent failed to " + + "start")) { + hasADCExtensionTestCompleted = true; } } } @@ -111,62 +106,13 @@ public class ADCExtensionTestCase extends PythonAgentIntegrationTest { }); adcExtensionTestThread.start(); - while (!hasCheckoutExtensionStarted || !hasCommitExtensionStarted) { + while (!hasADCExtensionTestCompleted) { // wait until the instance activated event is received. sleep(1000); } log.info("ADC Extension Test completed"); } - private void assertAgentActivation() { - Thread startupTestThread = new Thread(new Runnable() { - @Override - public void run() { - while (!eventReceiverInitiated) { - sleep(1000); - } - List<String> outputLines = new ArrayList<String>(); - while (!outputStream.isClosed()) { - List<String> newLines = getNewLines(outputLines, outputStream.toString()); - if (newLines.size() > 0) { - for (String line : newLines) { - if (line.contains("Subscribed to 'topology/#'")) { - sleep(2000); - // Send complete topology event - log.info("Publishing complete topology event..."); - Topology topology = createTestTopology(); - CompleteTopologyEvent completeTopologyEvent = new CompleteTopologyEvent(topology); - publishEvent(completeTopologyEvent); - log.info("Complete topology event published"); - - // Publish member initialized event - log.info("Publishing member initialized event..."); - MemberInitializedEvent memberInitializedEvent = new MemberInitializedEvent(SERVICE_NAME, - CLUSTER_ID, CLUSTER_INSTANCE_ID, MEMBER_ID, NETWORK_PARTITION_ID, PARTITION_ID); - publishEvent(memberInitializedEvent); - log.info("Member initialized event published"); - } - - // Send artifact updated event to activate the instance first - if (line.contains("Artifact repository found")) { - publishEvent(getArtifactUpdatedEventForPrivateRepo()); - log.info("Artifact updated event published"); - } - } - } - sleep(1000); - } - } - }); - startupTestThread.start(); - - while (!instanceStarted || !instanceActivated) { - // wait until the instance activated event is received. - // this will assert whether instance got activated within timeout period; no need for explicit assertions - sleep(2000); - } - } - public static ArtifactUpdatedEvent getArtifactUpdatedEventForPrivateRepo() { ArtifactUpdatedEvent privateRepoEvent = createTestArtifactUpdatedEvent(); privateRepoEvent.setRepoURL("https://bitbucket.org/testapache2211/testrepo.git";); http://git-wip-us.apache.org/repos/asf/stratos/blob/68fa338f/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/ADCTestCase.java ---------------------------------------------------------------------- diff --git a/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/ADCTestCase.java b/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/ADCTestCase.java index f3e797a..2c6cbcc 100755 --- a/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/ADCTestCase.java +++ b/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/ADCTestCase.java @@ -103,8 +103,8 @@ public class ADCTestCase extends PythonAgentIntegrationTest { tearDown(APPLICATION_PATH); } - - @Test(timeOut = ADC_TIMEOUT, groups = {"smoke"}) + @Test(timeOut = ADC_TIMEOUT, + groups = { "smoke" }) public void testADC() throws Exception { startCommunicatorThread(); assertAgentActivation(); @@ -119,8 +119,8 @@ public class ADCTestCase extends PythonAgentIntegrationTest { log.info("Running ADC Test thread..."); // Send artifact updated event publishEvent(getArtifactUpdatedEventForPrivateRepo()); - log.info("Publishing artifact updated event for repo: " + - getArtifactUpdatedEventForPrivateRepo().getRepoURL()); + log.info("Publishing artifact updated event for repo: " + getArtifactUpdatedEventForPrivateRepo() + .getRepoURL()); List<String> outputLines = new ArrayList<>(); while (!outputStream.isClosed() && !hasADCTestCompleted) { @@ -139,12 +139,11 @@ public class ADCTestCase extends PythonAgentIntegrationTest { } fileCreated = true; continue; - } - catch (IOException e) { + } catch (IOException e) { log.error("Could not create file", e); } } - if (fileCreated && line.contains("ArtifactUpdateTask completed") && !fileDeleted) { + if (fileCreated && line.contains("ArtifactUpdateTask end of iteration") && !fileDeleted) { if (!file.delete()) { throw new RuntimeException("Could not delete [file] " + file.getAbsolutePath()); } @@ -152,7 +151,8 @@ public class ADCTestCase extends PythonAgentIntegrationTest { continue; } // assert whether file deletion commit was pushed to remote repo - if (fileDeleted && line.contains("ArtifactUpdateTask completed") && !hasADCTestCompleted) { + if (fileDeleted && line.contains("ArtifactUpdateTask end of iteration") + && !hasADCTestCompleted) { boolean fileExists = new File(file.getAbsolutePath()).exists(); assertFalse("Deleted file has not been pushed to remote repo", fileExists); if (!fileExists) { @@ -197,10 +197,8 @@ public class ADCTestCase extends PythonAgentIntegrationTest { // Publish member initialized event log.info("Publishing member initialized event..."); - MemberInitializedEvent memberInitializedEvent = new MemberInitializedEvent( - SERVICE_NAME, CLUSTER_ID, CLUSTER_INSTANCE_ID, MEMBER_ID, NETWORK_PARTITION_ID, - PARTITION_ID - ); + MemberInitializedEvent memberInitializedEvent = new MemberInitializedEvent(SERVICE_NAME, + CLUSTER_ID, CLUSTER_INSTANCE_ID, MEMBER_ID, NETWORK_PARTITION_ID, PARTITION_ID); publishEvent(memberInitializedEvent); log.info("Member initialized event published"); } @@ -227,11 +225,9 @@ public class ADCTestCase extends PythonAgentIntegrationTest { private void assertRepoClone(ArtifactUpdatedEvent artifactUpdatedEvent) { File file = new File(APPLICATION_PATH + "/README.text"); - assertTrue("Git clone failed for repo [url] " + artifactUpdatedEvent.getRepoURL(), - file.exists()); + assertTrue("Git clone failed for repo [url] " + artifactUpdatedEvent.getRepoURL(), file.exists()); } - public static ArtifactUpdatedEvent getArtifactUpdatedEventForPublicRepo() { ArtifactUpdatedEvent publicRepoEvent = createTestArtifactUpdatedEvent(); publicRepoEvent.setRepoURL("https://bitbucket.org/testapache2211/opentestrepo1.git";); @@ -267,9 +263,8 @@ public class ADCTestCase extends PythonAgentIntegrationTest { AUTOSCALING_POLICY_NAME, APP_ID); service.addCluster(cluster); - Member member = new Member(service.getServiceName(), cluster.getClusterId(), MEMBER_ID, - CLUSTER_INSTANCE_ID, NETWORK_PARTITION_ID, PARTITION_ID, LoadBalancingIPType.Private, - System.currentTimeMillis()); + Member member = new Member(service.getServiceName(), cluster.getClusterId(), MEMBER_ID, CLUSTER_INSTANCE_ID, + NETWORK_PARTITION_ID, PARTITION_ID, LoadBalancingIPType.Private, System.currentTimeMillis()); member.setDefaultPrivateIP("10.0.0.1"); member.setDefaultPublicIP("20.0.0.1"); http://git-wip-us.apache.org/repos/asf/stratos/blob/68fa338f/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/ADCExtensionTestCase/extensions/py/checkout-job-handler.py ---------------------------------------------------------------------- diff --git a/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/ADCExtensionTestCase/extensions/py/checkout-job-handler.py b/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/ADCExtensionTestCase/extensions/py/checkout-job-handler.py index 3636e61..76acf11 100644 --- a/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/ADCExtensionTestCase/extensions/py/checkout-job-handler.py +++ b/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/ADCExtensionTestCase/extensions/py/checkout-job-handler.py @@ -16,14 +16,14 @@ # # ------------------------------------------------------------------------ -from plugins.contracts import ICartridgeAgentPlugin +from plugins.contracts import IArtifactCheckoutPlugin from modules.util.log import LogFactory from entity import * from modules.artifactmgt.git.agentgithandler import * -class CheckoutJobHandler(ICartridgeAgentPlugin): - def run_plugin(self, values): +class CheckoutJobHandler(IArtifactCheckoutPlugin): + def checkout(self, repo_info): log = LogFactory().get_log(__name__) try: log.info("Running extension for checkout job") http://git-wip-us.apache.org/repos/asf/stratos/blob/68fa338f/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/ADCExtensionTestCase/extensions/py/commit-job-handler.py ---------------------------------------------------------------------- diff --git a/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/ADCExtensionTestCase/extensions/py/commit-job-handler.py b/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/ADCExtensionTestCase/extensions/py/commit-job-handler.py index 8b68dfc..6c4c6a7 100644 --- a/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/ADCExtensionTestCase/extensions/py/commit-job-handler.py +++ b/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/ADCExtensionTestCase/extensions/py/commit-job-handler.py @@ -16,12 +16,12 @@ # # ------------------------------------------------------------------------ -from plugins.contracts import ICartridgeAgentPlugin +from plugins.contracts import IArtifactCommitPlugin from modules.util.log import LogFactory from entity import * -class CommitJobHandler(ICartridgeAgentPlugin): - def run_plugin(self, values): +class CommitJobHandler(IArtifactCommitPlugin): + def commit(self, repo_info): log = LogFactory().get_log(__name__) log.info("Running extension for commit job") http://git-wip-us.apache.org/repos/asf/stratos/blob/68fa338f/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/AgentStartupTestCase/agent.conf ---------------------------------------------------------------------- diff --git a/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/AgentStartupTestCase/agent.conf b/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/AgentStartupTestCase/agent.conf index abed7c9..35d462b 100755 --- a/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/AgentStartupTestCase/agent.conf +++ b/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/AgentStartupTestCase/agent.conf @@ -28,8 +28,8 @@ cep.stats.publisher.enabled =true lb.private.ip = lb.public.ip = enable.artifact.update =true -auto.commit =true -auto.checkout =true +auto.commit =false +auto.checkout =false artifact.update.interval =15 artifact.clone.retries =5 artifact.clone.interval =10 http://git-wip-us.apache.org/repos/asf/stratos/blob/68fa338f/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/AgentStartupTestCase/extensions/py/checkout-job-handler.py ---------------------------------------------------------------------- diff --git a/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/AgentStartupTestCase/extensions/py/checkout-job-handler.py b/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/AgentStartupTestCase/extensions/py/checkout-job-handler.py new file mode 100644 index 0000000..76acf11 --- /dev/null +++ b/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/AgentStartupTestCase/extensions/py/checkout-job-handler.py @@ -0,0 +1,34 @@ +# ------------------------------------------------------------------------ +# +# Copyright 2005-2015 WSO2, Inc. (http://wso2.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License +# +# ------------------------------------------------------------------------ + +from plugins.contracts import IArtifactCheckoutPlugin +from modules.util.log import LogFactory +from entity import * +from modules.artifactmgt.git.agentgithandler import * + + +class CheckoutJobHandler(IArtifactCheckoutPlugin): + def checkout(self, repo_info): + log = LogFactory().get_log(__name__) + try: + log.info("Running extension for checkout job") + repo_info = values['REPO_INFO'] + git_repo = AgentGitHandler.create_git_repo(repo_info) + AgentGitHandler.add_repo(git_repo) + except Exception as e: + log.exception("Error while executing CheckoutJobHandler extension: %s" % e) http://git-wip-us.apache.org/repos/asf/stratos/blob/68fa338f/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/AgentStartupTestCase/extensions/py/checkout-job-handler.yapsy-plugin ---------------------------------------------------------------------- diff --git a/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/AgentStartupTestCase/extensions/py/checkout-job-handler.yapsy-plugin b/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/AgentStartupTestCase/extensions/py/checkout-job-handler.yapsy-plugin new file mode 100644 index 0000000..fb383a7 --- /dev/null +++ b/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/AgentStartupTestCase/extensions/py/checkout-job-handler.yapsy-plugin @@ -0,0 +1,27 @@ +# ------------------------------------------------------------------------ +# +# Copyright 2005-2015 WSO2, Inc. (http://wso2.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License +# +# ------------------------------------------------------------------------ + +[Core] +Name = checkout-job-handler +Module = checkout-job-handler + +[Documentation] +Description = ArtifactCheckoutJob +Author = Stratos +Version = 0.1 +Website = stratos.apache.org http://git-wip-us.apache.org/repos/asf/stratos/blob/68fa338f/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/AgentStartupTestCase/extensions/py/commit-job-handler.py ---------------------------------------------------------------------- diff --git a/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/AgentStartupTestCase/extensions/py/commit-job-handler.py b/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/AgentStartupTestCase/extensions/py/commit-job-handler.py new file mode 100644 index 0000000..6c4c6a7 --- /dev/null +++ b/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/AgentStartupTestCase/extensions/py/commit-job-handler.py @@ -0,0 +1,27 @@ +# ------------------------------------------------------------------------ +# +# Copyright 2005-2015 WSO2, Inc. (http://wso2.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License +# +# ------------------------------------------------------------------------ + +from plugins.contracts import IArtifactCommitPlugin +from modules.util.log import LogFactory +from entity import * + + +class CommitJobHandler(IArtifactCommitPlugin): + def commit(self, repo_info): + log = LogFactory().get_log(__name__) + log.info("Running extension for commit job") http://git-wip-us.apache.org/repos/asf/stratos/blob/68fa338f/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/AgentStartupTestCase/extensions/py/commit-job-handler.yapsy-plugin ---------------------------------------------------------------------- diff --git a/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/AgentStartupTestCase/extensions/py/commit-job-handler.yapsy-plugin b/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/AgentStartupTestCase/extensions/py/commit-job-handler.yapsy-plugin new file mode 100644 index 0000000..90624b5 --- /dev/null +++ b/products/python-cartridge-agent/modules/integration/test-integration/src/test/resources/AgentStartupTestCase/extensions/py/commit-job-handler.yapsy-plugin @@ -0,0 +1,27 @@ +# ------------------------------------------------------------------------ +# +# Copyright 2005-2015 WSO2, Inc. (http://wso2.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License +# +# ------------------------------------------------------------------------ + +[Core] +Name = commit-job-handler +Module = commit-job-handler + +[Documentation] +Description = ArtifactCommitJob +Author = Stratos +Version = 0.1 +Website = stratos.apache.org
