Renamed python_cartridgeagent to cartridgeagent Wrote the maven-antrun-plugin execution to zip the cartridgeagent/cartridgeagent/* contents
Project: http://git-wip-us.apache.org/repos/asf/stratos/repo Commit: http://git-wip-us.apache.org/repos/asf/stratos/commit/c5adb7aa Tree: http://git-wip-us.apache.org/repos/asf/stratos/tree/c5adb7aa Diff: http://git-wip-us.apache.org/repos/asf/stratos/diff/c5adb7aa Branch: refs/heads/master Commit: c5adb7aa1fad02f569d3ccabca1768115c92c8ee Parents: 6550141 Author: Chamila de Alwis <[email protected]> Authored: Sat Nov 8 00:16:47 2014 +0530 Committer: lasinducharith <[email protected]> Committed: Sat Nov 8 08:46:24 2014 +0530 ---------------------------------------------------------------------- .../cartridgeagent/__init__.py | 0 .../cartridgeagent/cartridgeagent/__init__.py | 16 + .../cartridgeagent/cartridgeagent/agent.conf | 61 + .../cartridgeagent/cartridgeagent/agent.py | 349 ++++ .../cartridgeagent/cartridgeagent/logging.ini | 52 + .../cartridgeagent/modules/__init__.py | 16 + .../modules/artifactmgt/__init__.py | 17 + .../modules/artifactmgt/git/__init__.py | 17 + .../modules/artifactmgt/git/agentgithandler.py | 574 +++++++ .../modules/artifactmgt/git/gitrepository.py | 48 + .../artifactmgt/repositoryinformation.py | 37 + .../cartridgeagent/modules/config/__init__.py | 17 + .../config/cartridgeagentconfiguration.py | 346 ++++ .../modules/databridge/__init__.py | 17 + .../cartridgeagent/modules/databridge/agent.py | 225 +++ .../modules/databridge/thrift/__init__.py | 17 + .../databridge/thrift/gen/Data/__init__.py | 1 + .../databridge/thrift/gen/Data/constants.py | 8 + .../databridge/thrift/gen/Data/ttypes.py | 320 ++++ .../databridge/thrift/gen/Exception/__init__.py | 1 + .../thrift/gen/Exception/constants.py | 8 + .../databridge/thrift/gen/Exception/ttypes.py | 473 ++++++ .../ThriftEventTransmissionService-remote | 117 ++ .../ThriftEventTransmissionService.py | 1143 +++++++++++++ .../ThriftEventTransmissionService/__init__.py | 1 + .../ThriftEventTransmissionService/constants.py | 8 + .../ThriftEventTransmissionService/ttypes.py | 21 + .../ThriftSecureEventTransmissionService-remote | 131 ++ .../ThriftSecureEventTransmissionService.py | 1495 ++++++++++++++++++ .../__init__.py | 1 + .../constants.py | 8 + .../ttypes.py | 21 + .../modules/databridge/thrift/gen/__init__.py | 0 .../modules/databridge/thrift/publisher.py | 110 ++ .../modules/databridge/thrift/thrift/TSCons.py | 35 + .../databridge/thrift/thrift/TSerialization.py | 38 + .../databridge/thrift/thrift/TTornado.py | 153 ++ .../modules/databridge/thrift/thrift/Thrift.py | 170 ++ .../databridge/thrift/thrift/__init__.py | 20 + .../databridge/thrift/thrift/protocol/TBase.py | 81 + .../thrift/thrift/protocol/TBinaryProtocol.py | 261 +++ .../thrift/thrift/protocol/TCompactProtocol.py | 405 +++++ .../thrift/thrift/protocol/TJSONProtocol.py | 552 +++++++ .../thrift/thrift/protocol/TProtocol.py | 406 +++++ .../thrift/thrift/protocol/__init__.py | 20 + .../thrift/thrift/protocol/fastbinary.c | 1219 ++++++++++++++ .../thrift/thrift/server/THttpServer.py | 87 + .../thrift/thrift/server/TNonblockingServer.py | 346 ++++ .../thrift/thrift/server/TProcessPoolServer.py | 118 ++ .../databridge/thrift/thrift/server/TServer.py | 269 ++++ .../databridge/thrift/thrift/server/__init__.py | 20 + .../thrift/thrift/transport/THttpClient.py | 147 ++ .../thrift/thrift/transport/TSSLSocket.py | 214 +++ .../thrift/thrift/transport/TSocket.py | 176 +++ .../thrift/thrift/transport/TTransport.py | 330 ++++ .../thrift/thrift/transport/TTwisted.py | 221 +++ .../thrift/thrift/transport/TZlibTransport.py | 249 +++ .../thrift/thrift/transport/__init__.py | 20 + .../modules/datapublisher/__init__.py | 18 + .../modules/datapublisher/exception/__init__.py | 17 + .../exception/datapublisherexception.py | 33 + .../modules/datapublisher/logpublisher.py | 273 ++++ .../cartridgeagent/modules/event/__init__.py | 0 .../modules/event/instance/__init__.py | 16 + .../modules/event/instance/notifier/__init__.py | 17 + .../modules/event/instance/notifier/events.py | 77 + .../modules/event/instance/status/__init__.py | 17 + .../modules/event/instance/status/events.py | 98 ++ .../modules/event/tenant/__init__.py | 16 + .../modules/event/tenant/events.py | 147 ++ .../modules/event/topology/__init__.py | 17 + .../modules/event/topology/events.py | 280 ++++ .../modules/exception/__init__.py | 16 + .../exception/parameternotfoundexception.py | 35 + .../modules/extensions/__init__.py | 16 + .../extensions/abstractextensionhandler.py | 78 + .../extensions/defaultextensionhandler.py | 793 ++++++++++ .../modules/healthstatspublisher/__init__.py | 16 + .../abstracthealthstatisticspublisher.py | 62 + .../modules/healthstatspublisher/healthstats.py | 249 +++ .../modules/publisher/__init__.py | 16 + .../publisher/cartridgeagentpublisher.py | 165 ++ .../modules/subscriber/__init__.py | 17 + .../modules/subscriber/eventsubscriber.py | 96 ++ .../cartridgeagent/modules/tenant/__init__.py | 16 + .../modules/tenant/tenantcontext.py | 184 +++ .../cartridgeagent/modules/topology/__init__.py | 16 + .../modules/topology/topologycontext.py | 454 ++++++ .../cartridgeagent/modules/util/__init__.py | 16 + .../modules/util/asyncscheduledtask.py | 71 + .../modules/util/cartridgeagentconstants.py | 135 ++ .../modules/util/cartridgeagentutils.py | 168 ++ .../modules/util/extensionutils.py | 494 ++++++ .../cartridgeagent/modules/util/log.py | 55 + .../cartridgeagent/tests/__init__.py | 16 + .../cartridgeagent/tests/asynctest.txt | 1 + .../cartridgeagent/tests/test_util.py | 133 ++ .../pom.xml | 23 + .../python_cartridgeagent/__init__.py | 0 .../cartridgeagent/__init__.py | 16 - .../cartridgeagent/agent.conf | 61 - .../cartridgeagent/agent.py | 349 ---- .../cartridgeagent/logging.ini | 52 - .../cartridgeagent/modules/__init__.py | 16 - .../modules/artifactmgt/__init__.py | 17 - .../modules/artifactmgt/git/__init__.py | 17 - .../modules/artifactmgt/git/agentgithandler.py | 574 ------- .../modules/artifactmgt/git/gitrepository.py | 48 - .../artifactmgt/repositoryinformation.py | 37 - .../cartridgeagent/modules/config/__init__.py | 17 - .../config/cartridgeagentconfiguration.py | 346 ---- .../modules/databridge/__init__.py | 17 - .../cartridgeagent/modules/databridge/agent.py | 225 --- .../modules/databridge/thrift/__init__.py | 17 - .../databridge/thrift/gen/Data/__init__.py | 1 - .../databridge/thrift/gen/Data/constants.py | 8 - .../databridge/thrift/gen/Data/ttypes.py | 320 ---- .../databridge/thrift/gen/Exception/__init__.py | 1 - .../thrift/gen/Exception/constants.py | 8 - .../databridge/thrift/gen/Exception/ttypes.py | 473 ------ .../ThriftEventTransmissionService-remote | 117 -- .../ThriftEventTransmissionService.py | 1143 ------------- .../ThriftEventTransmissionService/__init__.py | 1 - .../ThriftEventTransmissionService/constants.py | 8 - .../ThriftEventTransmissionService/ttypes.py | 21 - .../ThriftSecureEventTransmissionService-remote | 131 -- .../ThriftSecureEventTransmissionService.py | 1495 ------------------ .../__init__.py | 1 - .../constants.py | 8 - .../ttypes.py | 21 - .../modules/databridge/thrift/gen/__init__.py | 0 .../modules/databridge/thrift/publisher.py | 110 -- .../modules/databridge/thrift/thrift/TSCons.py | 35 - .../databridge/thrift/thrift/TSerialization.py | 38 - .../databridge/thrift/thrift/TTornado.py | 153 -- .../modules/databridge/thrift/thrift/Thrift.py | 170 -- .../databridge/thrift/thrift/__init__.py | 20 - .../databridge/thrift/thrift/protocol/TBase.py | 81 - .../thrift/thrift/protocol/TBinaryProtocol.py | 261 --- .../thrift/thrift/protocol/TCompactProtocol.py | 405 ----- .../thrift/thrift/protocol/TJSONProtocol.py | 552 ------- .../thrift/thrift/protocol/TProtocol.py | 406 ----- .../thrift/thrift/protocol/__init__.py | 20 - .../thrift/thrift/protocol/fastbinary.c | 1219 -------------- .../thrift/thrift/server/THttpServer.py | 87 - .../thrift/thrift/server/TNonblockingServer.py | 346 ---- .../thrift/thrift/server/TProcessPoolServer.py | 118 -- .../databridge/thrift/thrift/server/TServer.py | 269 ---- .../databridge/thrift/thrift/server/__init__.py | 20 - .../thrift/thrift/transport/THttpClient.py | 147 -- .../thrift/thrift/transport/TSSLSocket.py | 214 --- .../thrift/thrift/transport/TSocket.py | 176 --- .../thrift/thrift/transport/TTransport.py | 330 ---- .../thrift/thrift/transport/TTwisted.py | 221 --- .../thrift/thrift/transport/TZlibTransport.py | 249 --- .../thrift/thrift/transport/__init__.py | 20 - .../modules/datapublisher/__init__.py | 18 - .../modules/datapublisher/exception/__init__.py | 17 - .../exception/datapublisherexception.py | 33 - .../modules/datapublisher/logpublisher.py | 273 ---- .../cartridgeagent/modules/event/__init__.py | 0 .../modules/event/instance/__init__.py | 16 - .../modules/event/instance/notifier/__init__.py | 17 - .../modules/event/instance/notifier/events.py | 77 - .../modules/event/instance/status/__init__.py | 17 - .../modules/event/instance/status/events.py | 98 -- .../modules/event/tenant/__init__.py | 16 - .../modules/event/tenant/events.py | 147 -- .../modules/event/topology/__init__.py | 17 - .../modules/event/topology/events.py | 280 ---- .../modules/exception/__init__.py | 16 - .../exception/parameternotfoundexception.py | 35 - .../modules/extensions/__init__.py | 16 - .../extensions/abstractextensionhandler.py | 78 - .../extensions/defaultextensionhandler.py | 793 ---------- .../modules/healthstatspublisher/__init__.py | 16 - .../abstracthealthstatisticspublisher.py | 62 - .../modules/healthstatspublisher/healthstats.py | 249 --- .../modules/publisher/__init__.py | 16 - .../publisher/cartridgeagentpublisher.py | 165 -- .../modules/subscriber/__init__.py | 17 - .../modules/subscriber/eventsubscriber.py | 96 -- .../cartridgeagent/modules/tenant/__init__.py | 16 - .../modules/tenant/tenantcontext.py | 184 --- .../cartridgeagent/modules/topology/__init__.py | 16 - .../modules/topology/topologycontext.py | 454 ------ .../cartridgeagent/modules/util/__init__.py | 16 - .../modules/util/asyncscheduledtask.py | 71 - .../modules/util/cartridgeagentconstants.py | 135 -- .../modules/util/cartridgeagentutils.py | 168 -- .../modules/util/extensionutils.py | 494 ------ .../cartridgeagent/modules/util/log.py | 55 - .../python_cartridgeagent/tests/__init__.py | 16 - .../python_cartridgeagent/tests/asynctest.txt | 1 - .../python_cartridgeagent/tests/test_util.py | 133 -- 195 files changed, 15600 insertions(+), 15577 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/stratos/blob/c5adb7aa/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/__init__.py ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/__init__.py b/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/__init__.py new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/stratos/blob/c5adb7aa/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/__init__.py ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/__init__.py b/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/__init__.py new file mode 100644 index 0000000..d216be4 --- /dev/null +++ b/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/stratos/blob/c5adb7aa/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/agent.conf ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/agent.conf b/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/agent.conf new file mode 100644 index 0000000..f3e2afb --- /dev/null +++ b/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/agent.conf @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[agent] +mb.ip =MB-IP +mb.port =MB-PORT +listen.address =LISTEN_ADDR +thrift.receiver.ip =CEP-IP +thrift.receiver.port =CEP-PORT +thrift.server.admin.username =CEP-ADMIN-USERNAME +thrift.server.admin.password =CEP-ADMIN-PASSWORD +param.file.path =/mnt/cartridgeagent/payload/launch-params +extensions.dir =/mnt/cartridgeagent/extensions +cep.stats.publisher.enabled =ENABLE_HEALTH_PUBLISHER +lb.private.ip =LB_PRIVATE_IP +lb.public.ip =LB_PUBLIC_IP +enable.artifact.update =ENABLE_ARTFCT_UPDATE +auto.commit =COMMIT_ENABLED +auto.checkout =CHECKOUT_ENABLED +artifact.update.interval =ARTFCT_UPDATE_INT +port.check.timeout =PORT_CHECK_TIMEOUT +enable.data.publisher =ENABLE-DATA-PUBLISHER +monitoring.server.ip =MONITORING-SERVER-IP +monitoring.server.port =MONITORING-SERVER-PORT +monitoring.server.secure.port =MONITORING-SERVER-SECURE-PORT +monitoring.server.admin.username =MONITORING-SERVER-ADMIN-USERNAME +monitoring.server.admin.password =MONITORING-SERVER-ADMIN-PASSWORD +log.file.paths =LOG_FILE_PATHS +super.tenant.repository.path =/repository/deployment/server/ +tenant.repository.path =/repository/tenants/ +extension.instance.started =instance-started.sh +extension.start.servers =start-servers.sh +extension.instance.activated =instance-activated.sh +extension.artifacts.updated =artifacts-updated.sh +extension.clean =clean.sh +extension.mount.volumes =mount_volumes.sh +extension.member.started =member-started.sh +extension.member.activated =member-activated.sh +extension.member.suspended =member-suspended.sh +extension.member.terminated =member-terminated.sh +extension.complete.topology =complete-topology.sh +extension.complete.tenant =complete-tenant.sh +extension.subscription.domain.added =subscription-domain-added.sh +extension.subscription.domain.removed =subscription-domain-removed.sh +extension.artifacts.copy =artifacts-copy.sh +extension.tenant.subscribed =tenant-subscribed.sh +extension.tenant.unsubscribed =tenant-unsubscribed.sh \ No newline at end of file http://git-wip-us.apache.org/repos/asf/stratos/blob/c5adb7aa/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/agent.py ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/agent.py b/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/agent.py new file mode 100644 index 0000000..4336990 --- /dev/null +++ b/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/agent.py @@ -0,0 +1,349 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import threading +import sys + +from modules.exception.parameternotfoundexception import ParameterNotFoundException +from modules.subscriber import eventsubscriber +from modules.publisher import cartridgeagentpublisher +from modules.event.instance.notifier.events import * +from modules.event.tenant.events import * +from modules.event.topology.events import * +from modules.tenant.tenantcontext import * +from modules.topology.topologycontext import * +from modules.datapublisher.logpublisher import * +from modules.config import cartridgeagentconfiguration +from modules.extensions import defaultextensionhandler + + +class CartridgeAgent(threading.Thread): + extension_handler = defaultextensionhandler.DefaultExtensionHandler() + + def __init__(self): + threading.Thread.__init__(self) + + mb_ip = cartridgeagentconfiguration.CartridgeAgentConfiguration().read_property(cartridgeagentconstants.MB_IP) + mb_port = cartridgeagentconfiguration.CartridgeAgentConfiguration().read_property(cartridgeagentconstants.MB_PORT) + + self.__instance_event_subscriber = eventsubscriber.EventSubscriber( + cartridgeagentconstants.INSTANCE_NOTIFIER_TOPIC, + mb_ip, + mb_port) + self.__tenant_event_subscriber = eventsubscriber.EventSubscriber( + cartridgeagentconstants.TENANT_TOPIC, + mb_ip, + mb_port) + self.__topology_event_subscriber = eventsubscriber.EventSubscriber( + cartridgeagentconstants.TOPOLOGY_TOPIC, + mb_ip, + mb_port) + + self.__tenant_context_initialized = False + + self.log_publish_manager = None + + self.terminated = False + + self.log = LogFactory().get_log(__name__) + + self.cartridge_agent_config = CartridgeAgentConfiguration() + + def run(self): + self.log.info("Starting Cartridge Agent...") + + #Check if required prpoerties are set + self.validate_required_properties() + + #Start instance notifier listener thread + self.subscribe_to_topics_and_register_listeners() + + #Start topology event receiver thread + self.register_topology_event_listeners() + + #Start tenant event receiver thread + self.register_tenant_event_listeners() + + #wait for intance spawned event + while not self.cartridge_agent_config.initialized: + self.log.debug("Waiting for Cartridge Agent to be initialized...") + time.sleep(1) + + #Execute instance started shell script + CartridgeAgent.extension_handler.on_instance_started_event() + + #Publish instance started event + cartridgeagentpublisher.publish_instance_started_event() + + #Execute start servers extension + try: + CartridgeAgent.extension_handler.start_server_extension() + except: + self.log.exception("Error processing start servers event") + + #Wait for all ports to be active + cartridgeagentutils.wait_until_ports_active( + self.cartridge_agent_config.listen_address, + self.cartridge_agent_config.ports, + int(self.cartridge_agent_config.read_property("port.check.timeout", critical=False)) + ) + + # check if artifact management is required before publishing instance activated event + repo_url = self.cartridge_agent_config.repo_url + if repo_url is None or str(repo_url).strip() == "": + self.log.info("No artifact repository found") + CartridgeAgent.extension_handler.on_instance_activated_event() + + cartridgeagentpublisher.publish_instance_activated_event() + + persistence_mappping_payload = self.cartridge_agent_config.persistence_mappings + if persistence_mappping_payload is not None: + CartridgeAgent.extension_handler.volume_mount_extension(persistence_mappping_payload) + + # start log publishing thread + if DataPublisherConfiguration.get_instance().enabled: + log_file_paths = self.cartridge_agent_config.log_file_paths + if log_file_paths is None: + self.log.exception("No valid log file paths found, no logs will be published") + else: + self.log_publish_manager = LogPublisherManager(log_file_paths) + self.log_publish_manager.start() + + while not self.terminated: + time.sleep(1) + + if DataPublisherConfiguration.get_instance().enabled: + self.log_publish_manager.terminate_all_publishers() + + def terminate(self): + """ + Allows the CartridgeAgent thread to be terminated + + :return: void + """ + self.terminated = True + + def validate_required_properties(self): + """ + Checks if required properties are set + :return: void + """ + #PARAM_FILE_PATH + try: + self.cartridge_agent_config.read_property(cartridgeagentconstants.PARAM_FILE_PATH) + except ParameterNotFoundException: + self.log.error("System property not found: %r" % cartridgeagentconstants.PARAM_FILE_PATH) + return + + #EXTENSIONS_DIR + try: + self.cartridge_agent_config.read_property(cartridgeagentconstants.EXTENSIONS_DIR) + except ParameterNotFoundException: + self.log.error("System property not found: %r" % cartridgeagentconstants.EXTENSIONS_DIR) + return + + def subscribe_to_topics_and_register_listeners(self): + self.log.debug("Starting instance notifier event message receiver thread") + + self.__instance_event_subscriber.register_handler("ArtifactUpdatedEvent", self.on_artifact_updated) + self.__instance_event_subscriber.register_handler("InstanceCleanupMemberEvent", self.on_instance_cleanup_member) + self.__instance_event_subscriber.register_handler("InstanceCleanupClusterEvent", self.on_instance_cleanup_cluster) + + self.__instance_event_subscriber.start() + self.log.info("Instance notifier event message receiver thread started") + + # wait till subscribed to continue + while not self.__instance_event_subscriber.is_subscribed(): + time.sleep(2) + + def on_artifact_updated(self, msg): + event_obj = ArtifactUpdatedEvent.create_from_json(msg.payload) + CartridgeAgent.extension_handler.on_artifact_updated_event(event_obj) + + def on_instance_cleanup_member(self, msg): + member_in_payload = self.cartridge_agent_config.member_id + event_obj = InstanceCleanupMemberEvent.create_from_json(msg.payload) + member_in_event = event_obj.member_id + if member_in_payload == member_in_event: + CartridgeAgent.extension_handler.on_instance_cleanup_member_event(event_obj) + + def on_instance_cleanup_cluster(self, msg): + event_obj = InstanceCleanupClusterEvent.create_from_json(msg.payload) + cluster_in_payload = self.cartridge_agent_config.cluster_id + cluster_in_event = event_obj.cluster_id + + if cluster_in_event == cluster_in_payload: + CartridgeAgent.extension_handler.on_instance_cleanup_cluster_event(event_obj) + + def register_topology_event_listeners(self): + self.log.debug("Starting topology event message receiver thread") + + self.__topology_event_subscriber.register_handler("MemberActivatedEvent", self.on_member_activated) + self.__topology_event_subscriber.register_handler("MemberTerminatedEvent", self.on_member_terminated) + self.__topology_event_subscriber.register_handler("MemberSuspendedEvent", self.on_member_suspended) + self.__topology_event_subscriber.register_handler("CompleteTopologyEvent", self.on_complete_topology) + self.__topology_event_subscriber.register_handler("MemberStartedEvent", self.on_member_started) + self.__topology_event_subscriber.register_handler("InstanceSpawnedEvent", self.on_instance_spawned) + + self.__topology_event_subscriber.start() + self.log.info("Cartridge Agent topology receiver thread started") + + def on_instance_spawned(self, msg): + self.log.debug("Instance spawned event received: %r" % msg.payload) + if self.cartridge_agent_config.initialized: + return + + event_obj = InstanceSpawnedEvent.create_from_json(msg.payload) + try: + CartridgeAgent.extension_handler.on_instance_spawned_event(event_obj) + except: + self.log.exception("Error processing instance spawned event") + + def on_member_activated(self, msg): + self.log.debug("Member activated event received: %r" % msg.payload) + if not self.cartridge_agent_config.initialized: + return + + event_obj = MemberActivatedEvent.create_from_json(msg.payload) + try: + CartridgeAgent.extension_handler.on_member_activated_event(event_obj) + except: + self.log.exception("Error processing member activated event") + + def on_member_terminated(self, msg): + self.log.debug("Member terminated event received: %r" % msg.payload) + if not self.cartridge_agent_config.initialized: + return + + event_obj = MemberTerminatedEvent.create_from_json(msg.payload) + try: + CartridgeAgent.extension_handler.on_member_terminated_event(event_obj) + except: + self.log.exception("Error processing member terminated event") + + def on_member_suspended(self, msg): + self.log.debug("Member suspended event received: %r" % msg.payload) + if not self.cartridge_agent_config.initialized: + return + + event_obj = MemberSuspendedEvent.create_from_json(msg.payload) + try: + CartridgeAgent.extension_handler.on_member_suspended_event(event_obj) + except: + self.log.exception("Error processing member suspended event") + + def on_complete_topology(self, msg): + if not self.cartridge_agent_config.initialized: + self.log.debug("Complete topology event received") + event_obj = CompleteTopologyEvent.create_from_json(msg.payload) + TopologyContext.update(event_obj.topology) + try: + CartridgeAgent.extension_handler.on_complete_topology_event(event_obj) + except: + self.log.exception("Error processing complete topology event") + else: + self.log.info("Complete topology event updating task disabled") + + def on_member_started(self, msg): + self.log.debug("Member started event received: %r" % msg.payload) + if not self.cartridge_agent_config.initialized: + return + + event_obj = MemberStartedEvent.create_from_json(msg.payload) + try: + CartridgeAgent.extension_handler.on_member_started_event(event_obj) + except: + self.log.exception("Error processing member started event") + + def register_tenant_event_listeners(self): + self.log.debug("Starting tenant event message receiver thread") + self.__tenant_event_subscriber.register_handler("SubscriptionDomainAddedEvent", self.on_subscription_domain_added) + self.__tenant_event_subscriber.register_handler("SubscriptionDomainsRemovedEvent", self.on_subscription_domain_removed) + self.__tenant_event_subscriber.register_handler("CompleteTenantEvent", self.on_complete_tenant) + self.__tenant_event_subscriber.register_handler("TenantSubscribedEvent", self.on_tenant_subscribed) + self.__tenant_event_subscriber.register_handler("TenantUnSubscribedEvent", self.on_tenant_unsubscribed) + + self.__tenant_event_subscriber.start() + self.log.info("Tenant event message receiver thread started") + + def on_subscription_domain_added(self, msg): + self.log.debug("Subscription domain added event received : %r" % msg.payload) + event_obj = SubscriptionDomainAddedEvent.create_from_json(msg.payload) + try: + CartridgeAgent.extension_handler.on_subscription_domain_added_event(event_obj) + except: + self.log.exception("Error processing subscription domains added event") + + def on_subscription_domain_removed(self, msg): + self.log.debug("Subscription domain removed event received : %r" % msg.payload) + event_obj = SubscriptionDomainRemovedEvent.create_from_json(msg.payload) + try: + CartridgeAgent.extension_handler.on_subscription_domain_removed_event(event_obj) + except: + self.log.exception("Error processing subscription domains removed event") + + def on_complete_tenant(self, msg): + if not self.__tenant_context_initialized: + self.log.debug("Complete tenant event received") + event_obj = CompleteTenantEvent.create_from_json(msg.payload) + TenantContext.update(event_obj.tenants) + + try: + CartridgeAgent.extension_handler.on_complete_tenant_event(event_obj) + self.__tenant_context_initialized = True + except: + self.log.exception("Error processing complete tenant event") + else: + self.log.info("Complete tenant event updating task disabled") + + def on_tenant_subscribed(self, msg): + self.log.debug("Tenant subscribed event received: %r" % msg.payload) + event_obj = TenantSubscribedEvent.create_from_json(msg.payload) + try: + CartridgeAgent.extension_handler.on_tenant_subscribed_event(event_obj) + except: + self.log.exception("Error processing tenant subscribed event") + + def on_tenant_unsubscribed(self, msg): + self.log.debug("Tenant unSubscribed event received: %r" % msg.payload) + event_obj = TenantUnsubscribedEvent.create_from_json(msg.payload) + try: + CartridgeAgent.extension_handler.on_tenant_unsubscribed_event(event_obj) + except: + self.log.exception("Error processing tenant unSubscribed event") + + +def uncaught_exception_mg(exctype, value, tb): + log = LogFactory().get_log(__name__) + log.exception("UNCAUGHT EXCEPTION:", value) + +def main(): + sys.excepthook = uncaught_exception_mg + cartridge_agent = CartridgeAgent() + log = LogFactory().get_log(__name__) + + try: + log.debug("Starting cartridge agent") + cartridge_agent.start() + except: + log.exception("Cartridge Agent Exception") + cartridge_agent.terminate() + + +if __name__ == "__main__": + main() http://git-wip-us.apache.org/repos/asf/stratos/blob/c5adb7aa/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/logging.ini ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/logging.ini b/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/logging.ini new file mode 100644 index 0000000..d71b541 --- /dev/null +++ b/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/logging.ini @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +[formatters] +keys=default + +[formatter_default] +format=[%(asctime)s] %(levelname)s {%(filename)s:%(funcName)s} - %(message)s +class=logging.Formatter + +[handlers] +keys=console, error_file, log_file + +[handler_console] +class=logging.StreamHandler +formatter=default +args=tuple() + +[handler_log_file] +class=logging.FileHandler +level=LOG_LEVEL +formatter=default +args=("agent.log", "w") + +[handler_error_file] +class=logging.FileHandler +level=ERROR +formatter=default +args=("error.log", "w") + +[loggers] +keys=root + +[logger_root] +level=LOG_LEVEL +formatter=default +handlers=console,error_file,log_file \ No newline at end of file http://git-wip-us.apache.org/repos/asf/stratos/blob/c5adb7aa/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/modules/__init__.py ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/modules/__init__.py b/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/modules/__init__.py new file mode 100644 index 0000000..d216be4 --- /dev/null +++ b/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/modules/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/stratos/blob/c5adb7aa/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/modules/artifactmgt/__init__.py ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/modules/artifactmgt/__init__.py b/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/modules/artifactmgt/__init__.py new file mode 100644 index 0000000..2456923 --- /dev/null +++ b/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/modules/artifactmgt/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + http://git-wip-us.apache.org/repos/asf/stratos/blob/c5adb7aa/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/modules/artifactmgt/git/__init__.py ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/modules/artifactmgt/git/__init__.py b/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/modules/artifactmgt/git/__init__.py new file mode 100644 index 0000000..2456923 --- /dev/null +++ b/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/modules/artifactmgt/git/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + http://git-wip-us.apache.org/repos/asf/stratos/blob/c5adb7aa/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/modules/artifactmgt/git/agentgithandler.py ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/modules/artifactmgt/git/agentgithandler.py b/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/modules/artifactmgt/git/agentgithandler.py new file mode 100644 index 0000000..7b5bbaf --- /dev/null +++ b/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/modules/artifactmgt/git/agentgithandler.py @@ -0,0 +1,574 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from threading import current_thread, Thread +from git import * +from gittle import Gittle, GittleAuth # GitPython and Gittle are both used at the time being for pros and cons of both +import urllib2 +import os +import pexpect +import subprocess + +from ... util.log import LogFactory +from ... util import cartridgeagentutils, extensionutils, cartridgeagentconstants +from gitrepository import GitRepository +from ... config import cartridgeagentconfiguration +from ... util.asyncscheduledtask import AbstractAsyncScheduledTask, ScheduledExecutor +from ... artifactmgt.repositoryinformation import RepositoryInformation + + +class AgentGitHandler: + """ + Handles all the git artifact management tasks related to a cartridge + """ + + log = LogFactory().get_log(__name__) + + SUPER_TENANT_ID = -1234 + SUPER_TENANT_REPO_PATH = "/repository/deployment/server/" + TENANT_REPO_PATH = "/repository/tenants/" + + extension_handler = None + + __git_repositories = {} + # (tenant_id => gitrepository.GitRepository) + + cartridge_agent_config = cartridgeagentconfiguration.CartridgeAgentConfiguration() + + @staticmethod + def checkout(repo_info): + """ + Checks out the code from the remote repository. + If local repository path is empty, a clone operation is done. + If there is a cloned repository already on the local repository path, a pull operation + will be performed. + If there are artifacts not in the repository already on the local repository path, + they will be added to a git repository, the remote url added as origin, and then + a pull operation will be performed. + + :param RepositoryInformation repo_info: The repository information object + :return: A tuple containing whether it was an initial clone or not, and the repository + context object + :rtype: tuple(bool, GitRepository) + """ + repo_context = AgentGitHandler.get_repo_context(repo_info.tenant_id) + if repo_context is not None: + #has been previously cloned, this is not the subscription run + subscribe_run = False + if AgentGitHandler.is_valid_git_repository(repo_context): + AgentGitHandler.log.debug("Existing git repository detected for tenant %r, no clone required" % repo_info.tenant_id) + AgentGitHandler.pull(repo_context) + else: + if not os.listdir(repo_context.local_repo_path): + #empty dir, clone + repo_context.repo = AgentGitHandler.clone(repo_info) + else: + #not empty + if AgentGitHandler.sync_initial_local_artifacts(repo_context): + AgentGitHandler.pull(repo_context) + else: + repo_context = None + else: + #subscribing run.. need to clone + subscribe_run = True + repo_context = AgentGitHandler.clone(repo_info) + + return subscribe_run, repo_context + + @staticmethod + def sync_initial_local_artifacts(repo_context): + #init git repo + AgentGitHandler.init(repo_context.local_repo_path) + + # add remote repos + return AgentGitHandler.add_remote(repo_context) + + @staticmethod + def add_remote(repo_context): + try: + #add origin remote + repo_context.repo.create_remote("origin", repo_context.repo_url) + #fetch branch details from origin + repo_context.repo.git.fetch() + #checkout master branch from origin/master as tracking + repo_context.repo.git.branch("-f", "--track", "master", "origin/master") + return True + except: + AgentGitHandler.log.exception("Error in adding remote origin %r for local repository %r" + % (repo_context.repo_url, repo_context.local_repo_path)) + return False + + @staticmethod + def init(path): + try: + repo = Gittle.init(path) + return repo + except: + AgentGitHandler.log.exception("Initializing local repo at %r failed" % path) + raise Exception("Initializing local repo at %r failed" % path) + + @staticmethod + def is_valid_git_repository(repo_context): + if repo_context.cloned: + return True + + for ref in repo_context.repo.refs: + try: + ref._get_object() + except ValueError: + return False + + return True + + @staticmethod + def pull(repo_context): + repo = Repo(repo_context.local_repo_path) + import agent + AgentGitHandler.extension_handler = agent.CartridgeAgent.extension_handler + try: + repo.git.checkout("master") + pull_output = repo.git.pull() + if "Already up-to-date." not in pull_output: + AgentGitHandler.log.debug("Artifacts were updated as a result of the pull operation, thread: %r - %r" % (current_thread().getName(), current_thread().ident)) + else: + AgentGitHandler.log.debug("Pull operation: Already up-to-date, thread: %r - %r" % (current_thread().getName(), current_thread().ident)) + + AgentGitHandler.extension_handler.on_artifact_update_scheduler_event(repo_context.tenant_id) + except GitCommandError as ex: + if "fatal: Could not read from remote repository." in ex: + #invalid configuration, need to delete and reclone + AgentGitHandler.log.warn("Git pull unsuccessful for tenant %r, invalid configuration. %r" % (repo_context.tenant_id, ex)) + cartridgeagentutils.delete_folder_tree(repo_context.local_repo_path) + AgentGitHandler.clone(RepositoryInformation( + repo_context.repo_url, + repo_context.repo_username, + repo_context.repo_password, + repo_context.local_repo_path, + repo_context.tenant_id, + repo_context.is_multitenant, + repo_context.commit_enabled + )) + AgentGitHandler.extension_handler.on_artifact_update_scheduler_event(repo_context.tenant_id) + elif "error: Your local changes to the following files would be overwritten by merge:" in ex: + #conflict error + AgentGitHandler.log.warn("Git pull unsuccessful for tenant %r, conflicts detected." % repo_context.tenant_id) + #raise ex + + """ + 0:'git pull' returned exit status 1: error: Your local changes to the following files would be overwritten by merge: + 1: README.md + 2: index.php + 3:Please, commit your changes or stash them before you can merge. + 4:Aborting + """ + conflict_list = [] + files_arr = str(ex).split("\n") + for file_index in range(1, len(files_arr) - 2): + file_name = files_arr[file_index].strip() + conflict_list.append(file_name) + AgentGitHandler.log.debug("Added the file path %r to checkout from the remote repository" % file_name) + + AgentGitHandler.checkout_individually(conflict_list, repo) + elif "fatal: unable to access " in ex: + #transport error + AgentGitHandler.log.exception("Accessing remote git repository %r failed for tenant %r" % (repo_context.repo_url, repo_context.tenant_id)) + else: + AgentGitHandler.log.exception("Git pull operation for tenant %r failed" % repo_context.tenant_id) + + @staticmethod + def checkout_individually(conflict_list, repo): + try: + for conflicted_file in conflict_list: + repo.git.checkout(conflicted_file) + AgentGitHandler.log.info("Checked out the conflicting files from the remote repository successfully") + except: + AgentGitHandler.log.exception("Checking out artifacts from index failed") + + @staticmethod + def clone(repo_info): + repo_context = None + try: + repo_context = AgentGitHandler.create_git_repo_context(repo_info) + #create the directory if it doesn't exist + if not os.path.isdir(repo_context.local_repo_path): + cartridgeagentutils.create_dir(repo_context.local_repo_path) + + #TODO: remove gittle stuff + #auth = AgentGitHandler.create_auth_configuration(repo_context) + auth = None + + if auth is not None: + # authentication is required, use Gittle + gittle_repo = Gittle.clone(repo_context.repo_url, repo_context.local_repo_path, auth=auth) + repo = Repo(repo_context.local_repo_path) + else: + # authentication is not required, use GitPython + repo = Repo.clone_from(repo_context.repo_url, repo_context.local_repo_path) + gittle_repo = Gittle(repo_context.local_repo_path) + + repo_context.cloned = True + repo_context.gittle_repo = gittle_repo + repo_context.repo = repo + AgentGitHandler.add_repo_context(repo_context) + AgentGitHandler.log.info("Git clone operation for tenant %r successful" % repo_context.tenant_id) + except urllib2.URLError: + AgentGitHandler.log.exception("Accessing remote git repository failed for tenant %r" % repo_context.tenant_id) + except OSError: + AgentGitHandler.log.exception("Permission denied for repository path for tenant %r" % repo_context.tenant_id) + except: + AgentGitHandler.log.exception("Git clone operation for tenant %r failed" % repo_context.tenant_id) + finally: + return repo_context + + @staticmethod + def create_auth_configuration(repo_context): + """ + Creates a GittleAuth object based on the type of authorization + :param GitRepository repo_context: The repository context object + :return: GittleAuth object or None if no authorization needed + :rtype: GittleAuth + """ + if repo_context.key_based_auth: + private_key = AgentGitHandler.get_private_key() + auth = GittleAuth(pkey=private_key) + elif repo_context.repo_username is not None and repo_context.repo_username.strip() != "" and \ + repo_context.repo_password is not None and repo_context.repo_password.strip() != "": + auth = GittleAuth(username=repo_context.repo_username, password=repo_context.repo_password) + else: + auth = None + + return auth + + @staticmethod + def get_private_key(): + """ + Returns a file handler to the private key path specified by Carbon or default if not specified + by Carbon + :return: The file object of the private key file + :rtype: file + """ + pkey_name = cartridgeagentutils.get_carbon_server_property("SshPrivateKeyName") + if pkey_name is None: + pkey_name = "wso2" + + pkey_path = cartridgeagentutils.get_carbon_server_property("SshPrivateKeyPath") + if pkey_path is None: + pkey_path = os.environ["HOME"] + "/.ssh" + + if pkey_path.endswith("/"): + pkey_ptr = pkey_path + pkey_name + else: + pkey_ptr = pkey_path + "/" + pkey_name + + pkey_file = open(pkey_ptr) + + return pkey_file + + + @staticmethod + def add_repo_context(repo_context): + AgentGitHandler.__git_repositories[repo_context.tenant_id] = repo_context + + @staticmethod + def get_repo_context(tenant_id): + """ + + :param int tenant_id: + :return: GitRepository object + :rtype: GitRepository + """ + if tenant_id in AgentGitHandler.__git_repositories: + return AgentGitHandler.__git_repositories[tenant_id] + + return None + + @staticmethod + def remove_repo_context(tenant_id): + if tenant_id in AgentGitHandler.__git_repositories: + del AgentGitHandler.__git_repositories[tenant_id] + + @staticmethod + def create_git_repo_context(repo_info): + repo_context = GitRepository() + repo_context.tenant_id = repo_info.tenant_id + repo_context.local_repo_path = AgentGitHandler.get_repo_path_for_tenant( + repo_info.tenant_id, repo_info.repo_path, repo_info.is_multitenant) + repo_context.repo_url = repo_info.repo_url + repo_context.repo_username = repo_info.repo_username + repo_context.repo_password = repo_info.repo_password + repo_context.is_multitenant = repo_info.is_multitenant + repo_context.commit_enabled = repo_info.commit_enabled + + if AgentGitHandler.is_key_based_auth(repo_info.repo_url, repo_info.tenant_id): + repo_context.key_based_auth = True + else: + repo_context.key_based_auth = False + + repo_context.cloned = False + + repo_context.repo = None + repo_context.gittle_repo = None + + return repo_context + + @staticmethod + def is_key_based_auth(repo_url, tenant_id): + """ + Checks if the given git repo has key based authentication + :param str repo_url: Git repository remote url + :param str tenant_id: Tenant ID + :return: True if key based, False otherwise + :rtype: bool + """ + if repo_url.startswith("http://";) or repo_url.startswith("https://";): + # username and password, not key based + return False + elif repo_url.startswith("git://github.com"): + # no auth required + return False + elif repo_url.startswith("ssh://") or "@" in repo_url: + # key based + return True + else: + AgentGitHandler.log.error("Invalid git URL provided for tenant " + tenant_id) + raise RuntimeError("Invalid git URL provided for tenant " + tenant_id) + + @staticmethod + def get_repo_path_for_tenant(tenant_id, git_local_repo_path, is_multitenant): + repo_path = "" + + if is_multitenant: + if tenant_id == AgentGitHandler.SUPER_TENANT_ID: + #super tenant, /repository/deploy/server/ + super_tenant_repo_path = AgentGitHandler.cartridge_agent_config.super_tenant_repository_path + #"app_path" + repo_path += git_local_repo_path + + if super_tenant_repo_path is not None and super_tenant_repo_path != "": + super_tenant_repo_path = super_tenant_repo_path if super_tenant_repo_path.startswith("/") else "/" + super_tenant_repo_path + super_tenant_repo_path = super_tenant_repo_path if super_tenant_repo_path.endswith("/") else super_tenant_repo_path + "/" + #"app_path/repository/deploy/server/" + repo_path += super_tenant_repo_path + else: + #"app_path/repository/deploy/server/" + repo_path += AgentGitHandler.SUPER_TENANT_REPO_PATH + + else: + #normal tenant, /repository/tenants/tenant_id + tenant_repo_path = AgentGitHandler.cartridge_agent_config.tenant_repository_path + #"app_path" + repo_path += git_local_repo_path + + if tenant_repo_path is not None and tenant_repo_path != "": + tenant_repo_path = tenant_repo_path if tenant_repo_path.startswith("/") else "/" + tenant_repo_path + tenant_repo_path = tenant_repo_path if tenant_repo_path.endswith("/") else tenant_repo_path + "/" + #"app_path/repository/tenants/244653444" + repo_path += tenant_repo_path + tenant_id + else: + #"app_path/repository/tenants/244653444" + repo_path += AgentGitHandler.TENANT_REPO_PATH + tenant_id + + #tenant_dir_path = git_local_repo_path + AgentGitHandler.TENANT_REPO_PATH + tenant_id + cartridgeagentutils.create_dir(repo_path) + else: + #not multi tenant, app_path + repo_path = git_local_repo_path + + AgentGitHandler.log.debug("Repo path returned : %r" % repo_path) + return repo_path + + @staticmethod + def commit(repo_info): + """ + Commits and pushes new artifacts to the remote repository + :param repo_info: + :return: + """ + tenant_id = repo_info.tenant_id + repo_context = AgentGitHandler.get_repo_context(tenant_id) + #check if modified + modified, unstaged_files = AgentGitHandler.get_unstaged_files(repo_context.local_repo_path) + + AgentGitHandler.log.debug("Modified: %r" % str(modified)) + + # TODO: check for unpushed commits and push them too + if not modified: + AgentGitHandler.log.debug("No changes detected in the local repository for tenant " + tenant_id) + return + + AgentGitHandler.stage_all(repo_context.local_repo_path) + + #commit to local repositpory + commit_message = "tenant " + tenant_id + "'s artifacts committed to local repo at " + repo_context.local_repo_path + commit_name="First Author" + commit_email="[email protected]" + #git config + (output, errors) = AgentGitHandler.execute_git_command(["config", "user.email", commit_email], repo_context.local_repo_path) + (output, errors) = AgentGitHandler.execute_git_command(["config", "user.name", commit_name], repo_context.local_repo_path) + + #commit + (output, errors) = AgentGitHandler.execute_git_command(["commit", "-m", commit_message], repo_context.local_repo_path) + if errors.strip() == "": + commit_hash = AgentGitHandler.find_between(output, "[master", "]").strip() + AgentGitHandler.log.debug("Committed artifacts for tenant : " + tenant_id + " : " + commit_hash) + else: + AgentGitHandler.log.exception("Committing artifacts to local repository failed for tenant " + tenant_id) + + #push to remote + try: + #TODO: check key based authentication + + push_op = pexpect.spawn('git push origin master', cwd=repo_context.local_repo_path) + #push_op.logfile = sys.stdout + push_op.expect("Username for .*") + push_op.sendline(repo_context.repo_username) + push_op.expect("Password for .*") + push_op.sendline(repo_context.repo_password) + # result = push_op.expect([commit_hash + " master -> master", "Authentication failed for"]) + # if result != 0: + # raise Exception + #TODO: handle push failure scenarios + #push_op.interact() + push_op.expect(pexpect.EOF) + + AgentGitHandler.log.debug("Pushed artifacts for tenant : " + tenant_id) + except: + AgentGitHandler.log.exception("Pushing artifacts to remote repository failed for tenant " + tenant_id) + + @staticmethod + def get_unstaged_files(repo_path): + + (output, errors) = AgentGitHandler.execute_git_command(["status"], repo_path=repo_path) + unstaged_files = {"modified":[], "untracked":[]} + + if "nothing to commit" in output: + return False, unstaged_files + + if "Changes not staged for commit" in output: + #there are modified files + modified_lines = output.split("\n\n")[2].split("\n") + for mod_line in modified_lines: + file_name = mod_line.split(":")[1].strip() + unstaged_files["modified"].append(file_name) + + if "Untracked files" in output: + #there are untracked files + untracked_files = output.split("Untracked files:")[1].split("\n\n")[1].split("\n") + for unt_line in untracked_files: + unstaged_files["untracked"].append(unt_line.strip()) + + return True, unstaged_files + + @staticmethod + def stage_all(repo_path): + (output, errors) = AgentGitHandler.execute_git_command(["add", "--all"], repo_path=repo_path) + return True if errors.strip() == "" else False + + @staticmethod + def find_between( s, first, last ): + try: + start = s.index( first ) + len( first ) + end = s.index( last, start ) + return s[start:end] + except ValueError: + return "" + + @staticmethod + def schedule_artifact_update_scheduled_task(repo_info, auto_checkout, auto_commit, update_interval): + repo_context = AgentGitHandler.get_repo_context(repo_info.tenant_id) + + if repo_context is None: + AgentGitHandler.log.error("Unable to schedule artifact sync task, repositoryContext null for tenant %r" % repo_info.tenant_id) + return + + if repo_context.scheduled_update_task is None: + artifact_update_task = ArtifactUpdateTask(repo_info, auto_checkout, auto_commit) + async_task = ScheduledExecutor(update_interval, artifact_update_task) + + repo_context.scheduled_update_task = async_task + async_task.start() + AgentGitHandler.log.info("Scheduled Artifact Synchronization Task for path %r" % repo_context.local_repo_path) + else: + AgentGitHandler.log.info("Artifact Synchronization Task for path %r already scheduled" % repo_context.local_repo_path) + + @staticmethod + def remove_repo(tenant_id): + repo_context = AgentGitHandler.get_repo_context(tenant_id) + + #stop artifact update task + repo_context.scheduled_update_task.terminate() + + #remove git contents + cartridgeagentutils.delete_folder_tree(repo_context.local_repo_path) + + AgentGitHandler.remove_repo_context(tenant_id) + + if tenant_id == -1234: + if AgentGitHandler.cartridge_agent_config.is_multitenant: + extensionutils.execute_copy_artifact_extension( + cartridgeagentconstants.SUPERTENANT_TEMP_PATH, + AgentGitHandler.cartridge_agent_config.app_path + "/repository/deployment/server/" + ) + + AgentGitHandler.log.info("git repository deleted for tenant %r" % repo_context.tenant_id) + + return True + + @staticmethod + def execute_git_command(command, repo_path): + """ + Executes the given command string with given environment parameters + :param list command: Command with arguments to be executed + :param dict[str, str] env_params: Environment variables to be used + :return: output and error string tuple, RuntimeError if errors occur + :rtype: tuple(str, str) + :exception: RuntimeError + """ + os_env = os.environ.copy() + + command.insert(0, "/usr/bin/git") + p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=os_env, cwd=repo_path) + output, errors = p.communicate() + if len(errors) > 0: + raise RuntimeError("Git Command execution failed: \n %r" % errors) + + return output, errors + + + +class ArtifactUpdateTask(AbstractAsyncScheduledTask): + """ + Checks if the autocheckout and autocommit are enabled and executes respective tasks + """ + + def __init__(self, repo_info, auto_checkout, auto_commit): + self.log = LogFactory().get_log(__name__) + self.repo_info = repo_info + self.auto_checkout = auto_checkout + self.auto_commit = auto_commit + + def execute_task(self): + try: + if self.auto_checkout: + self.log.debug("Running checkout job") + AgentGitHandler.checkout(self.repo_info) + except: + self.log.exception("Auto checkout task failed") + + if self.auto_commit: + self.log.debug("Running commit job") + AgentGitHandler.commit(self.repo_info) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/stratos/blob/c5adb7aa/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/modules/artifactmgt/git/gitrepository.py ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/modules/artifactmgt/git/gitrepository.py b/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/modules/artifactmgt/git/gitrepository.py new file mode 100644 index 0000000..d710d9b --- /dev/null +++ b/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/modules/artifactmgt/git/gitrepository.py @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class GitRepository: + """ + Represents a git repository inside a particular instance + """ + + def __init__(self): + self.repo_url = None + """ :type : str """ + self.local_repo_path = None + """ :type : str """ + self.cloned = False + """ :type : bool """ + self.repo = None + """ :type : git.repo.base.Repo """ + self.gittle_repo = None + """ :type : gittle.gittle.Gittle """ + self.tenant_id = None + """ :type : int """ + self.key_based_auth = False + """ :type : bool """ + self.repo_username = None + """ :type : str """ + self.repo_password = None + """ :type : str """ + self.is_multitenant = False + """ :type : bool """ + self.commit_enabled = False + """ :type : bool """ + self.scheduled_update_task = None + """:type : ScheduledExecutor """ \ No newline at end of file http://git-wip-us.apache.org/repos/asf/stratos/blob/c5adb7aa/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/modules/artifactmgt/repositoryinformation.py ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/modules/artifactmgt/repositoryinformation.py b/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/modules/artifactmgt/repositoryinformation.py new file mode 100644 index 0000000..b67eada --- /dev/null +++ b/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/modules/artifactmgt/repositoryinformation.py @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class RepositoryInformation: + """ + Holds repository information to be used in artifact management + """ + + def __init__(self, repo_url, repo_username, repo_password, repo_path, tenant_id, is_multitenant, commit_enabled): + self.repo_url = repo_url + """ :type : str """ + self.repo_username = repo_username + """ :type : str """ + self.repo_password = repo_password + """ :type : str """ + self.repo_path = repo_path + """ :type : str """ + self.tenant_id = tenant_id + """ :type : int """ + self.is_multitenant = is_multitenant + """ :type : bool """ + self.commit_enabled = commit_enabled + """ :type : bool """ \ No newline at end of file http://git-wip-us.apache.org/repos/asf/stratos/blob/c5adb7aa/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/modules/config/__init__.py ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/modules/config/__init__.py b/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/modules/config/__init__.py new file mode 100644 index 0000000..2456923 --- /dev/null +++ b/components/org.apache.stratos.python.cartridge.agent/cartridgeagent/cartridgeagent/modules/config/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +
