Repository: stratos Updated Branches: refs/heads/stratos-4.1.x 4a18fe100 -> 3043dc3ac
Fixing STRATOS-1568: Clone artifacts to a temporary location and mv in PCA Project: http://git-wip-us.apache.org/repos/asf/stratos/repo Commit: http://git-wip-us.apache.org/repos/asf/stratos/commit/3043dc3a Tree: http://git-wip-us.apache.org/repos/asf/stratos/tree/3043dc3a Diff: http://git-wip-us.apache.org/repos/asf/stratos/diff/3043dc3a Branch: refs/heads/stratos-4.1.x Commit: 3043dc3ac07546d33001918212a88d12df7f6be2 Parents: 4a18fe1 Author: Akila Perera <[email protected]> Authored: Fri Sep 18 00:16:03 2015 +0530 Committer: Akila Perera <[email protected]> Committed: Fri Sep 18 00:31:18 2015 +0530 ---------------------------------------------------------------------- .../modules/artifactmgt/git/agentgithandler.py | 54 ++++++++++---------- .../agent/integration/tests/ADCTestCase.java | 40 +++++++++++++-- 2 files changed, 62 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/stratos/blob/3043dc3a/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/modules/artifactmgt/git/agentgithandler.py ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/modules/artifactmgt/git/agentgithandler.py b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/modules/artifactmgt/git/agentgithandler.py index b8942a2..742f759 100644 --- a/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/modules/artifactmgt/git/agentgithandler.py +++ b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/modules/artifactmgt/git/agentgithandler.py @@ -20,6 +20,7 @@ import subprocess import shutil import time import os +import tempfile from git import * import constants @@ -28,7 +29,7 @@ from ...util.log import LogFactory from ...util.asyncscheduledtask import AbstractAsyncScheduledTask, ScheduledExecutor from ...artifactmgt.repository import Repository from exception import GitRepositorySynchronizationException - +from distutils.dir_util import copy_tree class AgentGitHandler: """ @@ -69,7 +70,7 @@ class AgentGitHandler: AgentGitHandler.log.debug("Git pull executed: [tenant-id] %s [repo-url] %s", git_repo.tenant_id, git_repo.repo_url) except GitRepositorySynchronizationException as e: - AgentGitHandler.log.debug("Warning: Git Pull operation failed: %s" % e.get_message()) + AgentGitHandler.log.debug("Warning: Git Pull operation failed: %s" % e) else: # not a valid repository, might've been corrupted. do a re-clone @@ -94,8 +95,9 @@ class AgentGitHandler: AgentGitHandler.log.debug("Git clone executed: [tenant-id] %s [repo-url] %s", git_repo.tenant_id, git_repo.repo_url) except GitRepositorySynchronizationException as e: - AgentGitHandler.log.warn("Warning: Git clone operation failed. Retrying...") + AgentGitHandler.log.error("Git clone operation failed: %s" % e) # If first git clone is failed, execute retry_clone operation + AgentGitHandler.log.info("Retrying Git clone operation...") AgentGitHandler.retry_clone(git_repo) return subscribe_run, updated @@ -198,30 +200,29 @@ class AgentGitHandler: @staticmethod def clone(git_repo): - if os.path.isdir(git_repo.local_repo_path) and os.listdir(git_repo.local_repo_path) != []: - # delete and recreate local repo path if not empty dir - AgentGitHandler.log.debug("Local repository path not empty. Cleaning.") - GitUtils.delete_folder_tree(git_repo.local_repo_path) - GitUtils.create_dir(git_repo.local_repo_path) - try: - Repo.clone_from(git_repo.repo_url, git_repo.local_repo_path) + # create a temporary location to clone + temp_repo_path = os.path.join(tempfile.gettempdir(), "pca-temp-" + git_repo.tenant_id) + if os.path.isdir(temp_repo_path) and os.listdir(temp_repo_path) != []: + GitUtils.delete_folder_tree(temp_repo_path) + GitUtils.create_dir(temp_repo_path) + # clone the repo to a temporary location first to avoid conflicts + Repo.clone_from(git_repo.repo_url, temp_repo_path) + + # move the cloned dir to application path + copy_tree(temp_repo_path, git_repo.local_repo_path) + AgentGitHandler.add_repo(git_repo) AgentGitHandler.log.info("Git clone operation for tenant %s successful" % git_repo.tenant_id) return git_repo except GitCommandError as e: - raise GitRepositorySynchronizationException("Error while cloning repository: %s" % e) + raise GitRepositorySynchronizationException("Error while cloning repository for tenant %s: %s" % ( + git_repo.tenant_id, e)) @staticmethod def retry_clone(git_repo): """Retry 'git clone' operation for defined number of attempts with defined intervals """ - if os.path.isdir(git_repo.local_repo_path) and os.listdir(git_repo.local_repo_path) != []: - # delete and recreate local repo path if not empty dir - AgentGitHandler.log.debug("Local repository path not empty. Cleaning.") - GitUtils.delete_folder_tree(git_repo.local_repo_path) - GitUtils.create_dir(git_repo.local_repo_path) - git_clone_successful = False # Read properties from agent.conf max_retry_attempts = int(Config.read_property(constants.ARTIFACT_CLONE_RETRIES, 5)) @@ -232,19 +233,17 @@ class AgentGitHandler: while git_clone_successful is False and retry_attempts < max_retry_attempts: try: retry_attempts += 1 - Repo.clone_from(git_repo.repo_url, git_repo.local_repo_path) - AgentGitHandler.add_repo(git_repo) + AgentGitHandler.clone(git_repo) AgentGitHandler.log.info( "Retrying attempt to git clone operation for tenant %s successful" % git_repo.tenant_id) git_clone_successful = True - except GitCommandError as e: - AgentGitHandler.log.warn("Retrying git clone attempt %s failed" % retry_attempts) + AgentGitHandler.log.warn("Retrying git clone attempt %s failed: %s" % (retry_attempts, e)) if retry_attempts < max_retry_attempts: time.sleep(retry_interval) - pass else: - raise GitRepositorySynchronizationException("Error while retrying git clone: %s" % e) + raise GitRepositorySynchronizationException("All attempts failed while retrying git clone: %s" + % e) @staticmethod def add_repo(git_repo): @@ -456,7 +455,7 @@ class AgentGitHandler: try: GitUtils.delete_folder_tree(git_repo.local_repo_path) except GitRepositorySynchronizationException as e: - AgentGitHandler.log.exception("Repository folder not deleted: %s" % e.get_message()) + AgentGitHandler.log.exception("Repository folder not deleted: %s" % e) AgentGitHandler.clear_repo(tenant_id) AgentGitHandler.log.info("git repository deleted for tenant %s" % git_repo.tenant_id) @@ -503,7 +502,7 @@ class ArtifactUpdateTask(AbstractAsyncScheduledTask): self.log.debug("Running commit job # %s" % self.invocation_count) AgentGitHandler.push(self.repo_info) except GitRepositorySynchronizationException as e: - self.log.exception("Auto commit failed: %s" % e.get_message()) + self.log.exception("Auto commit failed: %s" % e) if self.auto_checkout: try: @@ -511,7 +510,8 @@ class ArtifactUpdateTask(AbstractAsyncScheduledTask): AgentGitHandler.checkout(self.repo_info) # TODO: run updated scheduler extension except GitRepositorySynchronizationException as e: - self.log.exception("Auto checkout task failed: %s" % e.get_message()) + self.log.exception("Auto checkout task failed: %s" % e) + self.log.debug("ArtifactUpdateTask completed # %s" % self.invocation_count) class GitRepository: @@ -573,4 +573,4 @@ class GitUtils: shutil.rmtree(path) GitUtils.log.debug("Directory [%s] deleted." % path) except OSError as e: - raise GitRepositorySynchronizationException("Deletion of folder path %s failed: %s" % (path, e)) + raise GitRepositorySynchronizationException("Deletion of folder path %s failed: %s" % (path, e)) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/stratos/blob/3043dc3a/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/ADCTestCase.java ---------------------------------------------------------------------- diff --git a/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/ADCTestCase.java b/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/ADCTestCase.java index c836295..6277a55 100755 --- a/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/ADCTestCase.java +++ b/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/ADCTestCase.java @@ -32,12 +32,15 @@ import org.testng.annotations.Test; import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.Properties; import java.util.UUID; import static junit.framework.Assert.assertTrue; +import static org.testng.AssertJUnit.assertFalse; public class ADCTestCase extends PythonAgentIntegrationTest { private static final Log log = LogFactory.getLog(ADCTestCase.class); @@ -65,6 +68,25 @@ public class ADCTestCase extends PythonAgentIntegrationTest { // Set jndi.properties.dir system property for initializing event publishers and receivers System.setProperty("jndi.properties.dir", getCommonResourcesPath()); + // create a dummy directory structure in Git clone path to create a conflicting default artifacts scenario + File tempWorkDir = new File(APPLICATION_PATH); + if (!tempWorkDir.mkdir()) { + throw new Exception("Could not create dir at " + tempWorkDir.getAbsolutePath()); + } + File dummyDir = new File(APPLICATION_PATH + File.separator + "do_not_delete"); + File dummyFile = new File(APPLICATION_PATH + File.separator + "do_not_delete" + File.separator + + "do_not_delete"); + + if (!dummyDir.mkdir()) { + throw new Exception("Could not create dir at " + dummyDir.getAbsolutePath()); + } + if (!dummyFile.createNewFile()) { + throw new Exception("Could not create file at " + dummyFile.getAbsolutePath()); + } + // write some dummy string which should get replaced by remote repo's version + String msg = "This should not be visible"; + Files.write(Paths.get(dummyFile.getAbsolutePath()), msg.getBytes()); + // start Python agent with configurations provided in resource path super.setup(ADC_TIMEOUT); @@ -93,6 +115,7 @@ public class ADCTestCase extends PythonAgentIntegrationTest { File file = new File(APPLICATION_PATH + File.separator + artifactFileName); boolean fileCreated = false; boolean fileDeleted = false; + boolean repoCloned = false; log.info("Running ADC Test thread..."); // Send artifact updated event publishEvent(getArtifactUpdatedEventForPrivateRepo()); @@ -107,26 +130,32 @@ public class ADCTestCase extends PythonAgentIntegrationTest { if (line.contains("Git clone executed")) { log.info("Agent has completed git clone. Asserting the operation..."); assertRepoClone(getArtifactUpdatedEventForPrivateRepo()); - + repoCloned = true; + } + if (repoCloned && !fileCreated) { try { if (!file.createNewFile()) { throw new RuntimeException("Could not create [file] " + file.getAbsolutePath()); } fileCreated = true; + continue; } catch (IOException e) { log.error("Could not create file", e); } } - if (fileCreated && line.contains("Pushed artifacts for tenant")) { - log.info("ADC Test completed"); + if (fileCreated && line.contains("ArtifactUpdateTask completed") && !fileDeleted) { if (!file.delete()) { throw new RuntimeException("Could not delete [file] " + file.getAbsolutePath()); } fileDeleted = true; + continue; } - if (fileDeleted && line.contains("Git pull rebase executed in checkout job")) { - if (!file.exists()) { + // assert whether file deletion commit was pushed to remote repo + if (fileDeleted && line.contains("ArtifactUpdateTask completed") && !hasADCTestCompleted) { + boolean fileExists = new File(file.getAbsolutePath()).exists(); + assertFalse("Deleted file has not been pushed to remote repo", fileExists); + if (!fileExists) { hasADCTestCompleted = true; } } @@ -142,6 +171,7 @@ public class ADCTestCase extends PythonAgentIntegrationTest { // wait until the instance activated event is received. sleep(1000); } + log.info("ADC Test completed"); } private void assertAgentActivation() {
