Fix STRATOS-1549: PCA artifact checkout and commit jobs can be plugged in via a PCA extension
Project: http://git-wip-us.apache.org/repos/asf/stratos/repo Commit: http://git-wip-us.apache.org/repos/asf/stratos/commit/fe29fcbf Tree: http://git-wip-us.apache.org/repos/asf/stratos/tree/fe29fcbf Diff: http://git-wip-us.apache.org/repos/asf/stratos/diff/fe29fcbf Branch: refs/heads/stratos-4.1.x Commit: fe29fcbfcc60512780841b5d56b0c52e2f270c32 Parents: 1603452 Author: Akila Perera <[email protected]> Authored: Sat Oct 10 13:46:14 2015 +0530 Committer: Akila Perera <[email protected]> Committed: Sat Oct 10 13:47:16 2015 +0530 ---------------------------------------------------------------------- .../cartridge.agent/constants.py | 4 + .../extensions/py/ExtensionExecutor.py | 6 +- .../modules/artifactmgt/git/agentgithandler.py | 28 ++- .../integration/tests/ADCExtensionTestCase.java | 211 +++++++++++++++++++ .../agent/integration/tests/ADCTestCase.java | 8 +- .../integration/tests/AgentStartupTestCase.java | 4 +- .../tests/PythonAgentIntegrationTest.java | 16 +- 7 files changed, 259 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/stratos/blob/fe29fcbf/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/constants.py ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/constants.py b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/constants.py index a2709bf..301cd47 100644 --- a/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/constants.py +++ b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/constants.py @@ -109,6 +109,10 @@ MEMBER_STARTED_EVENT = "MemberStartedEvent" TENANT_SUBSCRIBED_EVENT = "TenantSubscribedEvent" APPLICATION_SIGNUP_REMOVAL_EVENT = "ApplicationSignUpRemovedEvent" +# ADC related extensions +ARTIFACT_CHECKOUT_JOB = "ArtifactCheckoutJob" +ARTIFACT_COMMIT_JOB = "ArtifactCommitJob" + CREATE_LVS_DUMMY_INTERFACE = "CreateLVSDummyInterface" PRIMARY = "PRIMARY" http://git-wip-us.apache.org/repos/asf/stratos/blob/fe29fcbf/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/extensions/py/ExtensionExecutor.py ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/extensions/py/ExtensionExecutor.py b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/extensions/py/ExtensionExecutor.py index 655afc4..7a5258e 100644 --- a/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/extensions/py/ExtensionExecutor.py +++ b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/extensions/py/ExtensionExecutor.py @@ -35,8 +35,8 @@ class ExtensionExecutor(ICartridgeAgentPlugin): try: output, errors = ExtensionExecutor.execute_script(event_name + ".sh", extension_values) - except OSError: - raise RuntimeError("Could not find an extension file for event %s" % event_name) + except Exception as e: + raise RuntimeError("Could not find an extension file for event %s %s" % (event_name, e)) if len(errors) > 0: raise RuntimeError("Extension execution failed for script %s: %s" % (event_name, errors)) @@ -60,4 +60,4 @@ class ExtensionExecutor(ICartridgeAgentPlugin): p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=extension_values) output, errors = p.communicate() - return output, errors \ No newline at end of file + return output, errors http://git-wip-us.apache.org/repos/asf/stratos/blob/fe29fcbf/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/modules/artifactmgt/git/agentgithandler.py ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/modules/artifactmgt/git/agentgithandler.py b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/modules/artifactmgt/git/agentgithandler.py index 208d6fe..0e79091 100644 --- a/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/modules/artifactmgt/git/agentgithandler.py +++ b/components/org.apache.stratos.python.cartridge.agent/src/main/python/cartridge.agent/cartridge.agent/modules/artifactmgt/git/agentgithandler.py @@ -62,6 +62,20 @@ class AgentGitHandler: subsequent calls or not :rtype: tuple(bool, bool) """ + plugins_for_event = Config.plugins.get(constants.ARTIFACT_CHECKOUT_JOB) + if plugins_for_event is not None and len(plugins_for_event) > 0: + if len(plugins_for_event) > 1: + for plugin_info in plugins_for_event: + AgentGitHandler.log.debug("Registered plugin name for commit job: %s" % plugin_info.name) + AgentGitHandler.log.error("More than one plugin registered for checkout job. Aborting...") + return + try: + plugin_values = {"REPO_INFO": repo_info} + plugins_for_event[0].plugin_object.run_plugin(plugin_values) + return + except Exception as e: + AgentGitHandler.log.exception("Error while executing checkout job extension: %s" % e) + git_repo = AgentGitHandler.create_git_repo(repo_info) if AgentGitHandler.get_repo(repo_info.tenant_id) is not None: # has been previously cloned, this is not the subscription run @@ -82,7 +96,7 @@ class AgentGitHandler: AgentGitHandler.log.debug("Executing git clone: [tenant-id] %s [repo-url] %s", git_repo.tenant_id, git_repo.repo_url) git_repo = AgentGitHandler.clone(git_repo) - updated = True + AgentGitHandler.add_repo(git_repo) AgentGitHandler.log.debug("Git clone executed: [tenant-id] %s [repo-url] %s", git_repo.tenant_id, git_repo.repo_url) else: @@ -317,6 +331,18 @@ class AgentGitHandler: :param repo_info: :return: """ + plugins_for_event = Config.plugins.get(constants.ARTIFACT_COMMIT_JOB) + if plugins_for_event is not None and len(plugins_for_event) > 0: + if len(plugins_for_event) > 1: + for plugin_info in plugins_for_event: + AgentGitHandler.log.debug("Registered plugin name for commit job: %s" % plugin_info.name) + AgentGitHandler.log.error("More than one plugin registered for commit job. Aborting...") + return + try: + plugins_for_event[0].plugin_object.run_plugin({}) + return + except Exception as e: + AgentGitHandler.log.exception("Error while executing commit job extension: %s " % e) git_repo = AgentGitHandler.get_repo(repo_info.tenant_id) if git_repo is None: http://git-wip-us.apache.org/repos/asf/stratos/blob/fe29fcbf/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/ADCExtensionTestCase.java ---------------------------------------------------------------------- diff --git a/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/ADCExtensionTestCase.java b/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/ADCExtensionTestCase.java new file mode 100644 index 0000000..5f321fe --- /dev/null +++ b/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/ADCExtensionTestCase.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.stratos.python.cartridge.agent.integration.tests; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.stratos.common.domain.LoadBalancingIPType; +import org.apache.stratos.messaging.domain.topology.*; +import org.apache.stratos.messaging.event.instance.notifier.ArtifactUpdatedEvent; +import org.apache.stratos.messaging.event.topology.CompleteTopologyEvent; +import org.apache.stratos.messaging.event.topology.MemberInitializedEvent; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +public class ADCExtensionTestCase extends PythonAgentIntegrationTest { + + public ADCExtensionTestCase() throws IOException { + } + + private static final Log log = LogFactory.getLog(ADCExtensionTestCase.class); + private static final int ADC_TIMEOUT = 300000; + private static final String APPLICATION_PATH = "/tmp/ADCExtensionTestCase"; + private static final String CLUSTER_ID = "tomcat.domain"; + private static final String DEPLOYMENT_POLICY_NAME = "deployment-policy-2"; + private static final String AUTOSCALING_POLICY_NAME = "autoscaling-policy-2"; + private static final String APP_ID = "application-2"; + private static final String MEMBER_ID = "tomcat.member-1"; + private static final String CLUSTER_INSTANCE_ID = "cluster-1-instance-1"; + private static final String NETWORK_PARTITION_ID = "network-partition-1"; + private static final String PARTITION_ID = "partition-1"; + private static final String TENANT_ID = "-1234"; + private static final String SERVICE_NAME = "tomcat"; + + private boolean hasADCExtensionTestCompleted = false; + private boolean hasCheckoutExtensionStarted = false; + private boolean hasCommitExtensionStarted = false; + + @BeforeMethod(alwaysRun = true) + public void setupADCExtensionTest() throws Exception { + log.info("Setting up ADCExtensionTestCase"); + // Set jndi.properties.dir system property for initializing event publishers and receivers + System.setProperty("jndi.properties.dir", getCommonResourcesPath()); + + // start Python agent with configurations provided in resource path + super.setup(ADC_TIMEOUT); + + // Simulate server socket + startServerSocket(8080); + } + + /** + * TearDown method for test method testPythonCartridgeAgent + */ + @AfterMethod(alwaysRun = true) + public void tearDownADCExtensionTest() { + // TODO: app path is duplicated in Java test and payload + tearDown(APPLICATION_PATH); + } + + @Test(timeOut = ADC_TIMEOUT) + public void testADC() throws Exception { + startCommunicatorThread(); + assertAgentActivation(); + Thread adcExtensionTestThread = new Thread(new Runnable() { + @Override + public void run() { + log.info("Running ADC Extension Test thread..."); + // Send artifact updated event + publishEvent(getArtifactUpdatedEventForPrivateRepo()); + log.info("Publishing artifact updated event for repo: " + getArtifactUpdatedEventForPrivateRepo() + .getRepoURL()); + + List<String> outputLines = new ArrayList<String>(); + while (!outputStream.isClosed() && !hasADCExtensionTestCompleted) { + List<String> newLines = getNewLines(outputLines, outputStream.toString()); + if (newLines.size() > 0) { + for (String line : newLines) { + if (line.contains("Running extension for checkout job")) { + hasCheckoutExtensionStarted = true; + } + if (line.contains("Running extension for commit job")) { + hasCommitExtensionStarted = true; + } + } + } + sleep(1000); + } + } + }); + adcExtensionTestThread.start(); + + while (!hasCheckoutExtensionStarted || !hasCommitExtensionStarted) { + // wait until the instance activated event is received. + sleep(1000); + } + log.info("ADC Extension Test completed"); + } + + private void assertAgentActivation() { + Thread startupTestThread = new Thread(new Runnable() { + @Override + public void run() { + while (!eventReceiverInitiated) { + sleep(1000); + } + List<String> outputLines = new ArrayList<String>(); + while (!outputStream.isClosed()) { + List<String> newLines = getNewLines(outputLines, outputStream.toString()); + if (newLines.size() > 0) { + for (String line : newLines) { + if (line.contains("Subscribed to 'topology/#'")) { + sleep(2000); + // Send complete topology event + log.info("Publishing complete topology event..."); + Topology topology = createTestTopology(); + CompleteTopologyEvent completeTopologyEvent = new CompleteTopologyEvent(topology); + publishEvent(completeTopologyEvent); + log.info("Complete topology event published"); + + // Publish member initialized event + log.info("Publishing member initialized event..."); + MemberInitializedEvent memberInitializedEvent = new MemberInitializedEvent(SERVICE_NAME, + CLUSTER_ID, CLUSTER_INSTANCE_ID, MEMBER_ID, NETWORK_PARTITION_ID, PARTITION_ID); + publishEvent(memberInitializedEvent); + log.info("Member initialized event published"); + } + + // Send artifact updated event to activate the instance first + if (line.contains("Artifact repository found")) { + publishEvent(getArtifactUpdatedEventForPrivateRepo()); + log.info("Artifact updated event published"); + } + } + } + sleep(1000); + } + } + }); + startupTestThread.start(); + + while (!instanceStarted || !instanceActivated) { + // wait until the instance activated event is received. + // this will assert whether instance got activated within timeout period; no need for explicit assertions + sleep(2000); + } + } + + public static ArtifactUpdatedEvent getArtifactUpdatedEventForPrivateRepo() { + ArtifactUpdatedEvent privateRepoEvent = createTestArtifactUpdatedEvent(); + privateRepoEvent.setRepoURL("https://bitbucket.org/testapache2211/testrepo.git";); + privateRepoEvent.setRepoUserName("testapache2211"); + privateRepoEvent.setRepoPassword("+to2qVW16jzy+Xb/zuafQQ=="); + return privateRepoEvent; + } + + private static ArtifactUpdatedEvent createTestArtifactUpdatedEvent() { + ArtifactUpdatedEvent artifactUpdatedEvent = new ArtifactUpdatedEvent(); + artifactUpdatedEvent.setClusterId(CLUSTER_ID); + artifactUpdatedEvent.setTenantId(TENANT_ID); + return artifactUpdatedEvent; + } + + /** + * Create test topology + * + * @return Topology object with mock information + */ + private Topology createTestTopology() { + Topology topology = new Topology(); + Service service = new Service(SERVICE_NAME, ServiceType.SingleTenant); + topology.addService(service); + + Cluster cluster = new Cluster(service.getServiceName(), CLUSTER_ID, DEPLOYMENT_POLICY_NAME, + AUTOSCALING_POLICY_NAME, APP_ID); + service.addCluster(cluster); + + Member member = new Member(service.getServiceName(), cluster.getClusterId(), MEMBER_ID, CLUSTER_INSTANCE_ID, + NETWORK_PARTITION_ID, PARTITION_ID, LoadBalancingIPType.Private, System.currentTimeMillis()); + + member.setDefaultPrivateIP("10.0.0.1"); + member.setDefaultPublicIP("20.0.0.1"); + Properties properties = new Properties(); + properties.setProperty("prop1", "value1"); + member.setProperties(properties); + member.setStatus(MemberStatus.Created); + cluster.addMember(member); + return topology; + } +} http://git-wip-us.apache.org/repos/asf/stratos/blob/fe29fcbf/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/ADCTestCase.java ---------------------------------------------------------------------- diff --git a/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/ADCTestCase.java b/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/ADCTestCase.java index 733a111..f3e797a 100755 --- a/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/ADCTestCase.java +++ b/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/ADCTestCase.java @@ -122,7 +122,7 @@ public class ADCTestCase extends PythonAgentIntegrationTest { log.info("Publishing artifact updated event for repo: " + getArtifactUpdatedEventForPrivateRepo().getRepoURL()); - List<String> outputLines = new ArrayList<String>(); + List<String> outputLines = new ArrayList<>(); while (!outputStream.isClosed() && !hasADCTestCompleted) { List<String> newLines = getNewLines(outputLines, outputStream.toString()); if (newLines.size() > 0) { @@ -181,7 +181,7 @@ public class ADCTestCase extends PythonAgentIntegrationTest { while (!eventReceiverInitiated) { sleep(1000); } - List<String> outputLines = new ArrayList<String>(); + List<String> outputLines = new ArrayList<>(); while (!outputStream.isClosed()) { List<String> newLines = getNewLines(outputLines, outputStream.toString()); if (newLines.size() > 0) { @@ -256,7 +256,7 @@ public class ADCTestCase extends PythonAgentIntegrationTest { /** * Create test topology * - * @return + * @return Topology object with mock information */ private Topology createTestTopology() { Topology topology = new Topology(); @@ -280,4 +280,4 @@ public class ADCTestCase extends PythonAgentIntegrationTest { cluster.addMember(member); return topology; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/stratos/blob/fe29fcbf/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/AgentStartupTestCase.java ---------------------------------------------------------------------- diff --git a/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/AgentStartupTestCase.java b/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/AgentStartupTestCase.java index 14f7bd9..ba8e8ec 100755 --- a/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/AgentStartupTestCase.java +++ b/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/AgentStartupTestCase.java @@ -181,7 +181,7 @@ public class AgentStartupTestCase extends PythonAgentIntegrationTest { /** * Create test topology * - * @return + * @return Topology object with mock information */ private Topology createTestTopology() { Topology topology = new Topology(); @@ -205,4 +205,4 @@ public class AgentStartupTestCase extends PythonAgentIntegrationTest { cluster.addMember(member); return topology; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/stratos/blob/fe29fcbf/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/PythonAgentIntegrationTest.java ---------------------------------------------------------------------- diff --git a/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/PythonAgentIntegrationTest.java b/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/PythonAgentIntegrationTest.java index 0cc8dc3..66b9290 100644 --- a/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/PythonAgentIntegrationTest.java +++ b/products/python-cartridge-agent/modules/integration/test-integration/src/test/java/org/apache/stratos/python/cartridge/agent/integration/tests/PythonAgentIntegrationTest.java @@ -61,7 +61,7 @@ public class PythonAgentIntegrationTest { protected final UUID PYTHON_AGENT_DIR_NAME = UUID.randomUUID(); protected Map<Integer, ServerSocket> serverSocketMap = new HashMap<>(); - protected Map<String, Executor> executorList = new HashMap<String, Executor>(); + protected Map<String, Executor> executorList = new HashMap<>(); protected int cepPort; protected int cepSSLPort; @@ -262,7 +262,7 @@ public class PythonAgentIntegrationTest { /** * Start server socket * - * @param port + * @param port Port number of server socket to be started */ protected void startServerSocket(final int port) { Thread socketThread = new Thread(new Runnable() { @@ -314,7 +314,7 @@ public class PythonAgentIntegrationTest { /** * Copy python agent distribution to a new folder, extract it and copy sample configuration files * - * @return + * @return Python cartridge agent home directory */ protected String setupPythonAgent() { try { @@ -415,7 +415,7 @@ public class PythonAgentIntegrationTest { /** * Execute shell command * - * @param commandText + * @param commandText Command string to be executed */ protected ByteArrayOutputStreamLocal executeCommand(final String commandText, int timeout) { final ByteArrayOutputStreamLocal outputStream = new ByteArrayOutputStreamLocal(); @@ -452,7 +452,7 @@ public class PythonAgentIntegrationTest { /** * Sleep current thread * - * @param time + * @param time Time to sleep in milli-seconds */ protected void sleep(long time) { try { @@ -467,7 +467,7 @@ public class PythonAgentIntegrationTest { * * @param currentOutputLines current output lines * @param output output - * @return + * @return new lines printed by Python agent process */ protected List<String> getNewLines(List<String> currentOutputLines, String output) { List<String> newLines = new ArrayList<String>(); @@ -487,7 +487,7 @@ public class PythonAgentIntegrationTest { /** * Publish messaging event * - * @param event + * @param event Event object to be published to message broker */ protected void publishEvent(Event event) { String topicName = MessagingUtil.getMessageTopicName(event); @@ -512,4 +512,4 @@ public class PythonAgentIntegrationTest { return closed; } } -} \ No newline at end of file +}
