This is an automated email from the ASF dual-hosted git repository.
aonishuk pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
commit 9103295a7a8a8da3efe0f1b96cf6d915bb8cbb1d
Author: Andrew Onishuk <aonis...@hortonworks.com>
AuthorDate: Wed Oct 10 15:30:43 2018 +0300
AMBARI-24758. Ambari-agent takes up too many cpu of perf (aonishuk)
---
.../src/main/python/ambari_agent/AmbariAgent.py | 2 ++
.../src/main/python/ambari_agent/Facter.py | 30 ++++++++++++++--------
.../ambari_agent/HostCheckReportFileHandler.py | 5 ++--
.../src/main/python/ambari_agent/HostCleanup.py | 3 ++-
.../src/main/python/ambari_agent/HostInfo.py | 5 ++--
.../main/python/ambari_agent/alerts/ams_alert.py | 4 +--
.../main/python/ambari_agent/alerts/base_alert.py | 3 ++-
.../python/ambari_agent/alerts/metric_alert.py | 4 +--
.../python/ambari_agent/alerts/script_alert.py | 3 ++-
ambari-agent/src/main/python/ambari_agent/main.py | 3 ++-
10 files changed, 38 insertions(+), 24 deletions(-)
diff --git a/ambari-agent/src/main/python/ambari_agent/AmbariAgent.py
b/ambari-agent/src/main/python/ambari_agent/AmbariAgent.py
index 6f5a14e..d3cbd14 100644
--- a/ambari-agent/src/main/python/ambari_agent/AmbariAgent.py
+++ b/ambari-agent/src/main/python/ambari_agent/AmbariAgent.py
@@ -51,6 +51,8 @@ def main():
mergedArgs = [PYTHON, AGENT_SCRIPT] + args
while status == AGENT_AUTO_RESTART_EXIT_CODE:
+ with open("/tmp/x" , "w") as fp:
+ fp.write(str(mergedArgs))
mainProcess = subprocess32.Popen(mergedArgs)
mainProcess.communicate()
status = mainProcess.returncode
diff --git a/ambari-agent/src/main/python/ambari_agent/Facter.py
b/ambari-agent/src/main/python/ambari_agent/Facter.py
index 3859ff2..366160d 100644
--- a/ambari-agent/src/main/python/ambari_agent/Facter.py
+++ b/ambari-agent/src/main/python/ambari_agent/Facter.py
@@ -372,6 +372,15 @@ class FacterWindows(Facter):
@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
class FacterLinux(Facter):
+ FIRST_WORDS_REGEXP = re.compile(r',$')
+ IFNAMES_REGEXP = re.compile("^\d")
+ SE_STATUS_REGEXP = re.compile('(enforcing|permissive|enabled)')
+ DIGITS_REGEXP = re.compile("\d+")
+ FREEMEM_REGEXP = re.compile("MemFree:.*?(\d+) .*")
+ TOTALMEM_REGEXP = re.compile("MemTotal:.*?(\d+) .*")
+ SWAPFREE_REGEXP = re.compile("SwapFree:.*?(\d+) .*")
+ SWAPTOTAL_REGEXP = re.compile("SwapTotal:.*?(\d+) .*")
+
# selinux command
GET_SE_LINUX_ST_CMD = "/usr/sbin/sestatus"
GET_IFCONFIG_SHORT_CMD = "ifconfig -s"
@@ -436,7 +445,7 @@ class FacterLinux(Facter):
try:
retcode, out, err = run_os_command(FacterLinux.GET_SE_LINUX_ST_CMD)
- se_status = re.search('(enforcing|permissive|enabled)', out)
+ se_status = FacterLinux.SE_STATUS_REGEXP.search(out)
if se_status:
return True
except OSError:
@@ -449,19 +458,18 @@ class FacterLinux(Facter):
if i.strip():
result = result + i.split()[0].strip() + ","
- result = re.sub(r',$', "", result)
+ result = FacterLinux.FIRST_WORDS_REGEXP.sub("", result)
return result
def return_ifnames_from_ip_link(self, ip_link_output):
list = []
- prog = re.compile("^\d")
for line in ip_link_output.splitlines():
- if prog.match(line):
+ if FacterLinux.IFNAMES_REGEXP.match(line):
list.append(line.split()[1].rstrip(":"))
return ",".join(list)
def data_return_first(self, patern, data):
- full_list = re.findall(patern, data)
+ full_list = patern.findall(data)
result = ""
if full_list:
result = full_list[0]
@@ -518,7 +526,7 @@ class FacterLinux(Facter):
# Return uptime seconds
def getUptimeSeconds(self):
try:
- return int(self.data_return_first("\d+", self.DATA_UPTIME_OUTPUT))
+ return int(self.data_return_first(FacterLinux.DIGITS_REGEXP,
self.DATA_UPTIME_OUTPUT))
except ValueError:
log.warn("Can't get an uptime value from
{0}".format(self.DATA_UPTIME_OUTPUT))
return 0
@@ -527,7 +535,7 @@ class FacterLinux(Facter):
def getMemoryFree(self):
#:memoryfree_mb => "MemFree",
try:
- return int(self.data_return_first("MemFree:.*?(\d+) .*",
self.DATA_MEMINFO_OUTPUT))
+ return int(self.data_return_first(FacterLinux.FREEMEM_REGEXP,
self.DATA_MEMINFO_OUTPUT))
except ValueError:
log.warn("Can't get free memory size from
{0}".format(self.DATA_MEMINFO_OUTPUT))
return 0
@@ -535,7 +543,7 @@ class FacterLinux(Facter):
# Return memorytotal
def getMemoryTotal(self):
try:
- return int(self.data_return_first("MemTotal:.*?(\d+) .*",
self.DATA_MEMINFO_OUTPUT))
+ return int(self.data_return_first(FacterLinux.TOTALMEM_REGEXP,
self.DATA_MEMINFO_OUTPUT))
except ValueError:
log.warn("Can't get total memory size from
{0}".format(self.DATA_MEMINFO_OUTPUT))
return 0
@@ -544,7 +552,7 @@ class FacterLinux(Facter):
def getSwapFree(self):
#:swapfree_mb => "SwapFree"
try:
- return int(self.data_return_first("SwapFree:.*?(\d+) .*",
self.DATA_MEMINFO_OUTPUT))
+ return int(self.data_return_first(FacterLinux.SWAPFREE_REGEXP,
self.DATA_MEMINFO_OUTPUT))
except ValueError:
log.warn("Can't get free swap memory size from
{0}".format(self.DATA_MEMINFO_OUTPUT))
return 0
@@ -553,7 +561,7 @@ class FacterLinux(Facter):
def getSwapSize(self):
#:swapsize_mb => "SwapTotal",
try:
- return int(self.data_return_first("SwapTotal:.*?(\d+) .*",
self.DATA_MEMINFO_OUTPUT))
+ return int(self.data_return_first(FacterLinux.SWAPTOTAL_REGEXP,
self.DATA_MEMINFO_OUTPUT))
except ValueError:
log.warn("Can't get total swap memory size from
{0}".format(self.DATA_MEMINFO_OUTPUT))
return 0
@@ -562,7 +570,7 @@ class FacterLinux(Facter):
def getMemorySize(self):
#:memorysize_mb => "MemTotal"
try:
- return int(self.data_return_first("MemTotal:.*?(\d+) .*",
self.DATA_MEMINFO_OUTPUT))
+ return int(self.data_return_first(FacterLinux.TOTALMEM_REGEXP,
self.DATA_MEMINFO_OUTPUT))
except ValueError:
log.warn("Can't get memory size from
{0}".format(self.DATA_MEMINFO_OUTPUT))
return 0
diff --git
a/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py
b/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py
index 940b597d..bf2e00f 100644
--- a/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py
+++ b/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py
@@ -28,7 +28,7 @@ import ConfigParser
HADOOP_ROOT_DIR = "/usr/hdp"
HADOOP_PERM_REMOVE_LIST = ["current"]
-HADOOP_ITEMDIR_REGEX = "(\d\.){3}\d-\d{4}"
+HADOOP_ITEMDIR_REGEXP = re.compile("(\d\.){3}\d-\d{4}")
logger = logging.getLogger(__name__)
class HostCheckReportFileHandler:
@@ -96,7 +96,6 @@ class HostCheckReportFileHandler:
if not os.path.exists(HADOOP_ROOT_DIR):
return []
- matcher = re.compile(HADOOP_ITEMDIR_REGEX) # pre-compile regexp
folder_content = os.listdir(HADOOP_ROOT_DIR)
remove_list = []
@@ -108,7 +107,7 @@ class HostCheckReportFileHandler:
remove_list.append(full_path)
remlist_items_count += 1
- if matcher.match(item) is not None:
+ if HADOOP_ITEMDIR_REGEXP.match(item) is not None:
remove_list.append(full_path)
remlist_items_count += 1
diff --git a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
index c6f5bff..07224ae 100644
--- a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
+++ b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
@@ -375,6 +375,7 @@ class HostCleanup:
pidList = []
cmd = "ps auxww"
(returncode, stdoutdata, stderrdata) = self.run_os_command(cmd)
+ line_regexp = re.compile("\s\s+")
if 0 == returncode and stdoutdata:
lines = stdoutdata.split('\n')
@@ -384,7 +385,7 @@ class HostCleanup:
identifier = identifier.strip()
if identifier in line:
logger.debug("Found " + line + " for " + identifier);
- line = re.sub("\s\s+" , " ", line) #replace multi spaces with
single space before calling the split
+ line = line_regexp.sub(" ", line) #replace multi spaces with
single space before calling the split
tokens = line.split(' ')
logger.debug(tokens)
logger.debug(len(tokens))
diff --git a/ambari-agent/src/main/python/ambari_agent/HostInfo.py
b/ambari-agent/src/main/python/ambari_agent/HostInfo.py
index 8307b23..650eca1 100644
--- a/ambari-agent/src/main/python/ambari_agent/HostInfo.py
+++ b/ambari-agent/src/main/python/ambari_agent/HostInfo.py
@@ -196,6 +196,8 @@ class HostInfoLinux(HostInfo):
THP_FILE_REDHAT = "/sys/kernel/mm/redhat_transparent_hugepage/enabled"
THP_FILE_UBUNTU = "/sys/kernel/mm/transparent_hugepage/enabled"
+ THP_REGEXP = re.compile("\[(.+)\]")
+
def __init__(self, config=None):
super(HostInfoLinux, self).__init__(config)
@@ -267,7 +269,6 @@ class HostInfoLinux(HostInfo):
logger.exception("Checking java processes failed")
def getTransparentHugePage(self):
- thp_regex = "\[(.+)\]"
file_name = None
if OSCheck.is_ubuntu_family():
file_name = self.THP_FILE_UBUNTU
@@ -277,7 +278,7 @@ class HostInfoLinux(HostInfo):
if file_name and os.path.isfile(file_name):
with open(file_name) as f:
file_content = f.read()
- return re.search(thp_regex, file_content).groups()[0]
+ return HostInfoLinux.THP_REGEXP.search(file_content).groups()[0]
else:
return ""
diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/ams_alert.py
b/ambari-agent/src/main/python/ambari_agent/alerts/ams_alert.py
index 4f59143..32ac725 100644
--- a/ambari-agent/src/main/python/ambari_agent/alerts/ams_alert.py
+++ b/ambari-agent/src/main/python/ambari_agent/alerts/ams_alert.py
@@ -22,7 +22,7 @@ import httplib
import imp
import time
import urllib
-from alerts.metric_alert import MetricAlert
+from alerts.metric_alert import MetricAlert, REALCODE_REGEXP
import ambari_simplejson as json
import logging
import re
@@ -212,7 +212,7 @@ def f(args):
self.minimum_value = metric_info['minimum_value']
if 'value' in metric_info:
- realcode = re.sub('(\{(\d+)\})', 'args[\g<2>][k]', metric_info['value'])
+ realcode = REALCODE_REGEXP.sub('(\{(\d+)\})', 'args[\g<2>][k]',
metric_info['value'])
self.custom_value_module = imp.new_module(str(uuid.uuid4()))
code = self.DYNAMIC_CODE_VALUE_TEMPLATE.format(realcode)
diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/base_alert.py
b/ambari-agent/src/main/python/ambari_agent/alerts/base_alert.py
index 1acad26..7b498c2 100644
--- a/ambari-agent/src/main/python/ambari_agent/alerts/base_alert.py
+++ b/ambari-agent/src/main/python/ambari_agent/alerts/base_alert.py
@@ -29,6 +29,7 @@ logger = logging.getLogger(__name__)
AlertUri = namedtuple('AlertUri', 'uri is_ssl_enabled')
class BaseAlert(object):
+ CONFIG_KEY_REGEXP = re.compile('{{(\S+?)}}')
# will force a kinit even if klist says there are valid tickets (4 hour
default)
_DEFAULT_KINIT_TIMEOUT = 14400000
@@ -214,7 +215,7 @@ class BaseAlert(object):
# parse {{foo-bar/baz}}/whatever/{{foobar-site/blah}}
# into
# ['foo-bar/baz', 'foobar-site/blah']
- placeholder_keys = re.findall("{{(\S+?)}}", key)
+ placeholder_keys = BaseAlert.CONFIG_KEY_REGEXP.findall(key)
# if none found, then return the original
if placeholder_keys is None or len(placeholder_keys) == 0:
diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
b/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
index 94da8d3..cd9350a 100644
--- a/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
+++ b/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
@@ -39,6 +39,7 @@ SECURITY_ENABLED_KEY = '{{cluster-env/security_enabled}}'
# default timeout
DEFAULT_CONNECTION_TIMEOUT = 5.0
+REALCODE_REGEXP = re.compile('(\{(\d+)\})')
class MetricAlert(BaseAlert):
@@ -282,14 +283,13 @@ from __future__ import division
def f(args):
return {0}
"""
-
def __init__(self, jmx_info):
self.custom_module = None
self.property_list = jmx_info['property_list']
self.property_map = {}
if 'value' in jmx_info:
- realcode = re.sub('(\{(\d+)\})', 'args[\g<2>]', jmx_info['value'])
+ realcode = REALCODE_REGEXP.sub('args[\g<2>]', jmx_info['value'])
self.custom_module = imp.new_module(str(uuid.uuid4()))
code = self.DYNAMIC_CODE_TEMPLATE.format(realcode)
diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py
b/ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py
index 073d6c0..fbebd7e 100644
--- a/ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py
+++ b/ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py
@@ -31,6 +31,7 @@ from ambari_commons.constants import AGENT_TMP_DIR
logger = logging.getLogger(__name__)
class ScriptAlert(BaseAlert):
+ PATH_TO_SCRIPT_REGEXP = re.compile(r'((.*)services(.*)package)')
def __init__(self, alert_meta, alert_source_meta, config):
@@ -112,7 +113,7 @@ class ScriptAlert(BaseAlert):
# try to get basedir for scripts
# it's needed for server side scripts to properly use resource management
- matchObj = re.match( r'((.*)services(.*)package)', self.path_to_script)
+ matchObj = ScriptAlert.PATH_TO_SCRIPT_REGEXP.match(self.path_to_script)
if matchObj:
basedir = matchObj.group(1)
with Environment(basedir, tmp_dir=AGENT_TMP_DIR,
logger=logging.getLogger('alerts')) as env:
diff --git a/ambari-agent/src/main/python/ambari_agent/main.py
b/ambari-agent/src/main/python/ambari_agent/main.py
index 924b2d6..492f94c 100644
--- a/ambari-agent/src/main/python/ambari_agent/main.py
+++ b/ambari-agent/src/main/python/ambari_agent/main.py
@@ -88,6 +88,7 @@ import time
import locale
import platform
import ConfigParser
+import signal
import resource
from logging.handlers import SysLogHandler
import AmbariConfig
@@ -364,7 +365,7 @@ def run_threads(initializer_module):
initializer_module.action_queue.start()
while not initializer_module.stop_event.is_set():
- time.sleep(0.1)
+ signal.pause()
initializer_module.action_queue.interrupt()
