This is an automated email from the ASF dual-hosted git repository. av pushed a commit to branch ignite-ducktape in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/ignite-ducktape by this push: new dd6f2f7 IGNITE-13829: added rotate log (#8578) dd6f2f7 is described below commit dd6f2f7e9bc8f360bfd77b1891b1d193147745f5 Author: Sergei Ryzhov <s.vi.ryz...@gmail.com> AuthorDate: Fri Dec 18 15:27:02 2020 +0300 IGNITE-13829: added rotate log (#8578) --- modules/ducktests/tests/ignitetest/services/ignite.py | 2 +- .../ducktests/tests/ignitetest/services/ignite_app.py | 2 +- .../tests/ignitetest/services/utils/ignite_aware.py | 19 ++++++++++++++++--- .../ignitetest/services/utils/ignite_persistence.py | 6 ++++-- .../tests/ignitetest/services/utils/ignite_spec.py | 13 ++++++------- .../tests/ignitetest/tests/discovery_test.py | 6 +++--- 6 files changed, 31 insertions(+), 17 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/services/ignite.py b/modules/ducktests/tests/ignitetest/services/ignite.py index e5ec3fa..7923dc7 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite.py +++ b/modules/ducktests/tests/ignitetest/services/ignite.py @@ -83,7 +83,7 @@ def get_event_time(service, log_node, log_pattern, from_the_beginning=True, time backoff_sec=0.3) _, stdout, _ = log_node.account.ssh_client.exec_command( - "grep '%s' %s" % (log_pattern, IgniteAwareService.STDOUT_STDERR_CAPTURE)) + "grep '%s' %s" % (log_pattern, log_node.log_file)) return datetime.strptime(re.match("^\\[[^\\[]+\\]", stdout.read().decode("utf-8")).group(), "[%Y-%m-%d %H:%M:%S,%f]") diff --git a/modules/ducktests/tests/ignitetest/services/ignite_app.py b/modules/ducktests/tests/ignitetest/services/ignite_app.py index bb93725..8833582 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite_app.py +++ b/modules/ducktests/tests/ignitetest/services/ignite_app.py @@ -102,7 +102,7 @@ class IgniteApplicationService(IgniteAwareService): for node in self.nodes: output = node.account.ssh_capture( - "grep '%s' %s" % (name + "->", self.STDOUT_STDERR_CAPTURE), allow_fail=False) + "grep '%s' %s" % (name + "->", node.log_file), allow_fail=False) for line in output: res.append(re.search("%s(.*)%s" % (name + "->", "<-"), line).group(1)) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index 43fca2f..06fbe32 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -83,6 +83,8 @@ class IgniteAwareService(BackgroundThreadService, IgnitePersistenceAware, metacl def start_node(self, node): self.init_persistent(node) + self.__update_node_log_file(node) + super().start_node(node) wait_until(lambda: self.alive(node), timeout_sec=10) @@ -188,7 +190,7 @@ class IgniteAwareService(BackgroundThreadService, IgnitePersistenceAware, metacl # pylint: disable=W0613 def _worker(self, idx, node): - cmd = self.spec.command + cmd = self.spec.command(node.log_file) self.logger.debug("Attempting to start Application Service on %s with command: %s" % (str(node.account), cmd)) @@ -201,7 +203,8 @@ class IgniteAwareService(BackgroundThreadService, IgnitePersistenceAware, metacl """ return len(self.pids(node)) > 0 - def await_event_on_node(self, evt_message, node, timeout_sec, from_the_beginning=False, backoff_sec=5): + @staticmethod + def await_event_on_node(evt_message, node, timeout_sec, from_the_beginning=False, backoff_sec=5): """ Await for specific event message in a node's log file. :param evt_message: Event message. @@ -211,7 +214,7 @@ class IgniteAwareService(BackgroundThreadService, IgnitePersistenceAware, metacl :param backoff_sec: Number of seconds to back off between each failure to meet the condition before checking again. """ - with monitor_log(node, self.STDOUT_STDERR_CAPTURE, from_the_beginning) as monitor: + with monitor_log(node, node.log_file, from_the_beginning) as monitor: monitor.wait_until(evt_message, timeout_sec=timeout_sec, backoff_sec=backoff_sec, err_msg="Event [%s] was not triggered on '%s' in %d seconds" % (evt_message, node.name, timeout_sec)) @@ -351,3 +354,13 @@ class IgniteAwareService(BackgroundThreadService, IgnitePersistenceAware, metacl Reads current netfilter settings on the node for debugging purposes. """ return str(node.account.ssh_client.exec_command("sudo iptables -L -n")[1].read(), sys.getdefaultencoding()) + + def __update_node_log_file(self, node): + """ + Update the node log file. + """ + cnt = list(node.account.ssh_capture(f'ls {self.LOGS_DIR} | ' + f'grep -E "^console_[0-9]*.log$" | ' + f'wc -l', callback=int))[0] + + node.log_file = self.STDOUT_STDERR_CAPTURE.replace('.log', f'_{cnt + 1}.log') diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_persistence.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_persistence.py index 7cdc03c..4e018cc 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_persistence.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_persistence.py @@ -28,12 +28,13 @@ class PersistenceAware: """ # Root directory for persistent output PERSISTENT_ROOT = "/mnt/service" - STDOUT_STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "console.log") TEMP_DIR = os.path.join(PERSISTENT_ROOT, "tmp") + LOGS_DIR = os.path.join(PERSISTENT_ROOT, "logs") + STDOUT_STDERR_CAPTURE = os.path.join(LOGS_DIR, "console.log") logs = { "console_log": { - "path": STDOUT_STDERR_CAPTURE, + "path": LOGS_DIR, "collect_default": True } } @@ -45,6 +46,7 @@ class PersistenceAware: """ node.account.mkdirs(self.PERSISTENT_ROOT) node.account.mkdirs(self.TEMP_DIR) + node.account.mkdirs(self.LOGS_DIR) class IgnitePersistenceAware(PersistenceAware): diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py index ac664f3..6b9ba9f 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py @@ -74,7 +74,6 @@ class IgniteSpec(metaclass=ABCMeta): return IgniteClientConfigTemplate() return IgniteServerConfigTemplate() - @property @abstractmethod def command(self): """ @@ -100,14 +99,14 @@ class IgniteNodeSpec(IgniteSpec, IgnitePersistenceAware): """ Spec to run ignite node """ - @property - def command(self): + # pylint: disable=W0221 + def command(self, stdout_stderr): cmd = "%s %s %s %s 2>&1 | tee -a %s &" % \ (self._envs(), self.path.script("ignite.sh"), self._jvm_opts(), self.CONFIG_FILE, - self.STDOUT_STDERR_CAPTURE) + stdout_stderr) return cmd @@ -123,14 +122,14 @@ class IgniteApplicationSpec(IgniteSpec, IgnitePersistenceAware): def _app_args(self): return ",".join(self.args) - @property - def command(self): + # pylint: disable=W0221 + def command(self, stdout_stderr): cmd = "%s %s %s %s 2>&1 | tee -a %s &" % \ (self._envs(), self.path.script("ignite.sh"), self._jvm_opts(), self._app_args(), - self.STDOUT_STDERR_CAPTURE) + stdout_stderr) return cmd diff --git a/modules/ducktests/tests/ignitetest/tests/discovery_test.py b/modules/ducktests/tests/ignitetest/tests/discovery_test.py index eb0d250..658591e 100644 --- a/modules/ducktests/tests/ignitetest/tests/discovery_test.py +++ b/modules/ducktests/tests/ignitetest/tests/discovery_test.py @@ -244,13 +244,13 @@ class DiscoveryTest(IgniteTest): def _check_failed_number(self, failed_nodes, survived_node): """Ensures number of failed nodes is correct.""" - cmd = "grep '%s' %s | wc -l" % (node_failed_event_pattern(), IgniteAwareService.STDOUT_STDERR_CAPTURE) + cmd = "grep '%s' %s | wc -l" % (node_failed_event_pattern(), survived_node.log_file) failed_cnt = int(str(survived_node.account.ssh_client.exec_command(cmd)[1].read(), sys.getdefaultencoding())) if failed_cnt != len(failed_nodes): failed = str(survived_node.account.ssh_client.exec_command( - "grep '%s' %s" % (node_failed_event_pattern(), IgniteAwareService.STDOUT_STDERR_CAPTURE))[1].read(), + "grep '%s' %s" % (node_failed_event_pattern(), survived_node.log_file))[1].read(), sys.getdefaultencoding()) self.logger.warn("Node '%s' (%s) has detected the following failures:%s%s" % ( @@ -263,7 +263,7 @@ class DiscoveryTest(IgniteTest): """Ensures only target nodes failed""" for service in [srv for srv in self.test_context.services if isinstance(srv, IgniteAwareService)]: for node in [srv_node for srv_node in service.nodes if srv_node not in failed_nodes]: - cmd = "grep -i '%s' %s | wc -l" % ("local node segmented", IgniteAwareService.STDOUT_STDERR_CAPTURE) + cmd = "grep -i '%s' %s | wc -l" % ("local node segmented", node.log_file) failed = str(node.account.ssh_client.exec_command(cmd)[1].read(), sys.getdefaultencoding())