Fixed issues in git commit with username password auth Added initial methods for git handling with popen and pexpect
Project: http://git-wip-us.apache.org/repos/asf/stratos/repo Commit: http://git-wip-us.apache.org/repos/asf/stratos/commit/9a4e3a84 Tree: http://git-wip-us.apache.org/repos/asf/stratos/tree/9a4e3a84 Diff: http://git-wip-us.apache.org/repos/asf/stratos/diff/9a4e3a84 Branch: refs/heads/master Commit: 9a4e3a845b582ef0f0982d0de4b282f685d83750 Parents: 514a7e3 Author: Chamila de Alwis <[email protected]> Authored: Sat Oct 25 04:04:11 2014 +0530 Committer: Chamila de Alwis <[email protected]> Committed: Sat Oct 25 04:04:11 2014 +0530 ---------------------------------------------------------------------- .../modules/artifactmgt/git/agentgithandler.py | 115 ++++++++++++++----- .../extensions/defaultextensionhandler.py | 2 +- .../modules/util/extensionutils.py | 2 +- .../cartridgeagent/testgit.py | 2 +- 4 files changed, 91 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/stratos/blob/9a4e3a84/tools/python_cartridgeagent/cartridgeagent/modules/artifactmgt/git/agentgithandler.py ---------------------------------------------------------------------- diff --git a/tools/python_cartridgeagent/cartridgeagent/modules/artifactmgt/git/agentgithandler.py b/tools/python_cartridgeagent/cartridgeagent/modules/artifactmgt/git/agentgithandler.py index d6381a0..d5d085c 100644 --- a/tools/python_cartridgeagent/cartridgeagent/modules/artifactmgt/git/agentgithandler.py +++ b/tools/python_cartridgeagent/cartridgeagent/modules/artifactmgt/git/agentgithandler.py @@ -20,6 +20,8 @@ from git import * from gittle import Gittle, GittleAuth # GitPython and Gittle are both used at the time being for pros and cons of both import urllib2 import os +import pexpect +import subprocess from ... util.log import LogFactory from ... util import cartridgeagentutils, extensionutils, cartridgeagentconstants @@ -205,6 +207,7 @@ class AgentGitHandler: if not os.path.isdir(repo_context.local_repo_path): cartridgeagentutils.create_dir(repo_context.local_repo_path) + #TODO: remove gittle stuff auth = AgentGitHandler.create_auth_configuration(repo_context) if auth is not None: @@ -396,52 +399,90 @@ class AgentGitHandler: """ tenant_id = repo_info.tenant_id repo_context = AgentGitHandler.get_repo_context(tenant_id) - gittle_repo = repo_context.gittle_repo - try: - modified = True if gittle_repo.modified_unstaged_files.count > 0 else False - except OSError: - # removed files - modified = True + #check if modified + modified, unstaged_files = AgentGitHandler.get_unstaged_files(repo_context.local_repo_path) + + AgentGitHandler.log.debug("Modified: %r" % str(modified)) if not modified: AgentGitHandler.log.debug("No changes detected in the local repository for tenant " + tenant_id) return - gittle_repo.stage(gittle_repo.untracked_files) - gittle_repo.stage(gittle_repo.removed_files) - gittle_repo.stage(gittle_repo.modified_unstaged_files) + AgentGitHandler.stage_all(repo_context.local_repo_path) #commit to local repositpory commit_message = "tenant " + tenant_id + "'s artifacts committed to local repo at " + repo_context.local_repo_path - - try: - commit_hash = gittle_repo.commit(name="First Author", email="[email protected]", message=commit_message) + commit_name="First Author" + commit_email="[email protected]" + #git config + (output, errors) = AgentGitHandler.execute_git_command(["config", "user.email", commit_email], repo_context.local_repo_path) + (output, errors) = AgentGitHandler.execute_git_command(["config", "user.name", commit_name], repo_context.local_repo_path) + + #commit + (output, errors) = AgentGitHandler.execute_git_command(["commit", "-m", commit_message], repo_context.local_repo_path) + if errors.strip() == "": + commit_hash = AgentGitHandler.find_between(output, "[master", "]").strip() AgentGitHandler.log.debug("Committed artifacts for tenant : " + tenant_id + " : " + commit_hash) - except: + else: AgentGitHandler.log.exception("Committing artifacts to local repository failed for tenant " + tenant_id) #push to remote try: - repo = repo_context.repo #TODO: check key based authentication - credentialed_remote_url = AgentGitHandler.get_credentialed_remote_url(repo_context) - push_remote = repo.create_remote('push_remote', credentialed_remote_url) - push_remote.push() + + push_op = pexpect.spawn('git push origin master', cwd=repo_context.local_repo_path) + #push_op.logfile = sys.stdout + push_op.expect("Username for .*") + push_op.sendline(repo_context.repo_username) + push_op.expect("Password for .*") + push_op.sendline(repo_context.repo_password) + # result = push_op.expect([commit_hash + " master -> master", "Authentication failed for"]) + # if result != 0: + # raise Exception + #TODO: handle push failure scenarios + push_op.interact() + AgentGitHandler.log.debug("Pushed artifacts for tenant : " + tenant_id) except: AgentGitHandler.log.exception("Pushing artifacts to remote repository failed for tenant " + tenant_id) @staticmethod - def get_credentialed_remote_url(repo_context): - """ - Creates a remote url including the credentials - :param repo_context: - :return: - """ - username = repo_context.repo_username - password = repo_context.repo_password + def get_unstaged_files(repo_path): + + (output, errors) = AgentGitHandler.execute_git_command(["status"], repo_path=repo_path) + unstaged_files = {"modified":[], "untracked":[]} + + if "nothing to commit" in output: + return False, unstaged_files - raise NotImplementedError + if "Changes not staged for commit" in output: + #there are modified files + modified_lines = output.split("\n\n")[2].split("\n") + for mod_line in modified_lines: + file_name = mod_line.split(":")[1].strip() + unstaged_files["modified"].append(file_name) + + if "Untracked files" in output: + #there are untracked files + untracked_files = output.split("Untracked files:")[1].split("\n\n")[1].split("\n") + for unt_line in untracked_files: + unstaged_files["untracked"].append(unt_line.strip()) + + return True, unstaged_files + + @staticmethod + def stage_all(repo_path): + (output, errors) = AgentGitHandler.execute_git_command(["add", "--all"], repo_path=repo_path) + return True if errors.strip() == "" else False + + @staticmethod + def find_between( s, first, last ): + try: + start = s.index( first ) + len( first ) + end = s.index( last, start ) + return s[start:end] + except ValueError: + return "" @staticmethod def schedule_artifact_update_scheduled_task(repo_info, auto_checkout, auto_commit, update_interval): @@ -452,7 +493,6 @@ class AgentGitHandler: return if repo_context.scheduled_update_task is None: - #TODO: make thread safe artifact_update_task = ArtifactUpdateTask(repo_info, auto_checkout, auto_commit) async_task = ScheduledExecutor(update_interval, artifact_update_task) @@ -485,6 +525,27 @@ class AgentGitHandler: return True + @staticmethod + def execute_git_command(command, repo_path): + """ + Executes the given command string with given environment parameters + :param list command: Command with arguments to be executed + :param dict[str, str] env_params: Environment variables to be used + :return: output and error string tuple, RuntimeError if errors occur + :rtype: tuple(str, str) + :exception: RuntimeError + """ + os_env = os.environ.copy() + + command.insert(0, "/usr/bin/git") + p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=os_env, cwd=repo_path) + output, errors = p.communicate() + if len(errors) > 0: + raise RuntimeError("Git Command execution failed: \n %r" % errors) + + return output, errors + + class ArtifactUpdateTask(AbstractAsyncScheduledTask): """ http://git-wip-us.apache.org/repos/asf/stratos/blob/9a4e3a84/tools/python_cartridgeagent/cartridgeagent/modules/extensions/defaultextensionhandler.py ---------------------------------------------------------------------- diff --git a/tools/python_cartridgeagent/cartridgeagent/modules/extensions/defaultextensionhandler.py b/tools/python_cartridgeagent/cartridgeagent/modules/extensions/defaultextensionhandler.py index 07107da..bbbc100 100644 --- a/tools/python_cartridgeagent/cartridgeagent/modules/extensions/defaultextensionhandler.py +++ b/tools/python_cartridgeagent/cartridgeagent/modules/extensions/defaultextensionhandler.py @@ -100,7 +100,7 @@ class DefaultExtensionHandler(AbstractExtensionHandler): try: update_interval = int( - self.cartridge_agent_config.read_property(cartridgeagentconstants.ARTIFACT_UPDATE_INTERVAL, False)) + self.cartridge_agent_config.read_property(cartridgeagentconstants.ARTIFACT_UPDATE_INTERVAL)) except ParameterNotFoundException: self.log.exception("Invalid artifact sync interval specified ") update_interval = 10 http://git-wip-us.apache.org/repos/asf/stratos/blob/9a4e3a84/tools/python_cartridgeagent/cartridgeagent/modules/util/extensionutils.py ---------------------------------------------------------------------- diff --git a/tools/python_cartridgeagent/cartridgeagent/modules/util/extensionutils.py b/tools/python_cartridgeagent/cartridgeagent/modules/util/extensionutils.py index 6c58852..22639d3 100644 --- a/tools/python_cartridgeagent/cartridgeagent/modules/util/extensionutils.py +++ b/tools/python_cartridgeagent/cartridgeagent/modules/util/extensionutils.py @@ -468,7 +468,7 @@ def get_lb_member_ip(lb_cluster_id): return None -def execute_command(command, env_params=None): +def execute_command(command, env_params=None, cwd=None): """ Executes the given command string with given environment parameters :param str command: Command with arguments to be executed http://git-wip-us.apache.org/repos/asf/stratos/blob/9a4e3a84/tools/python_cartridgeagent/cartridgeagent/testgit.py ---------------------------------------------------------------------- diff --git a/tools/python_cartridgeagent/cartridgeagent/testgit.py b/tools/python_cartridgeagent/cartridgeagent/testgit.py index c683b56..0b38f83 100644 --- a/tools/python_cartridgeagent/cartridgeagent/testgit.py +++ b/tools/python_cartridgeagent/cartridgeagent/testgit.py @@ -19,7 +19,7 @@ from modules.extensions.defaultextensionhandler import * from modules.event.instance.notifier.events import ArtifactUpdatedEvent -event_msg = '{"clusterId":"php.php.domain","repoPassword":"","repoURL":"https://github.com/chamilad/NeWoice","tenantId":"-1234","commitEnabled":false}' +event_msg = '{"clusterId":"newoice.php.domain","repoUserName":"chamilad","repoPassword":"eng3lhimm3lsonn3","repoURL":"https://github.com/chamilad/NeWoice","tenantId":"-1234","commitEnabled":false}' event = ArtifactUpdatedEvent.create_from_json(event_msg) extension_handler = DefaultExtensionHandler()
