This is an automated email from the ASF dual-hosted git repository.
bbender pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode-native.git
The following commit(s) were added to refs/heads/develop by this push:
new 176c737 Performance improvements for gnmsg script (#828)
176c737 is described below
commit 176c73728752f7779cc5f793e513799186e8dd88
Author: Blake Bender <bben...@pivotal.io>
AuthorDate: Thu Jul 1 10:49:29 2021 -0700
Performance improvements for gnmsg script (#828)
- Compiling regex every time we check a line for matches is silly, made
the regexes member variables of the decoder classes and only compile
them once.
- Short-circuited regex checking by doing a "sanity check" using a
substring search for each line before using regex. Substring search
is much, much cheaper than re.search().
---
tools/gnmsg/client_message_decoder.py | 49 +++++++++------
tools/gnmsg/server_message_decoder.py | 113 +++++++++++++++++++---------------
2 files changed, 95 insertions(+), 67 deletions(-)
diff --git a/tools/gnmsg/client_message_decoder.py
b/tools/gnmsg/client_message_decoder.py
index 45e88f7..3e2bdf3 100644
--- a/tools/gnmsg/client_message_decoder.py
+++ b/tools/gnmsg/client_message_decoder.py
@@ -77,12 +77,21 @@ class ClientMessageDecoder(DecoderBase):
"USER_CREDENTIAL_MESSAGE",
]
- def get_send_trace_parts_v911(self, line, parts):
- result = False
- expression = re.compile(
+ self.security_trace_expression_ = re.compile(
+ r"(\d\d\d\d\/\d\d\/\d\d
\d\d:\d\d:\d\d\.\d+).*([\d|a-f|A-F|x|X]+)\]\s*TcrMessage::addSecurityPart\s*\[(0x[\d|a-f|A-F]*).*length\s*=\s*(\d+)\s*,\s*encrypted\s+ID\s*=\s*([\d|a-f|A-F]+)"
+ )
+
+ self.send_trace_expression_v911_ = re.compile(
r"(\d\d\d\d\/\d\d\/\d\d
\d\d:\d\d:\d\d\.\d+).*TcrConnection::send:\s*\[([\d|a-f|A-F|x|X]+).*sending
request to endpoint.*bytes:\s*([\d| ]+)"
)
- match = expression.search(line)
+
+ self.send_trace_expression_base_ = re.compile(
+ r"(\d\d\d\d\/\d\d\/\d\d
\d\d:\d\d:\d\d\.\d+).+:\d+\s+([\d|a-f|A-F|x|X]+)\]\s*TcrConnection::send:\s*\[([\d|a-f|A-F|x|X]+).*sending
request to endpoint.*bytes:\s*([\d|a-f|A-F]+)"
+ )
+
+ def get_send_trace_parts_v911(self, line, parts):
+ result = False
+ match = self.send_trace_expression_v911_.search(line)
if match:
parts.append(parser.parse(match.group(1)))
# TODO: Revisit parsing TID here if we ever see a v9 client log
again
@@ -95,10 +104,7 @@ class ClientMessageDecoder(DecoderBase):
def get_send_trace_parts_base(self, line, parts):
result = False
- expression = re.compile(
- r"(\d\d\d\d\/\d\d\/\d\d
\d\d:\d\d:\d\d\.\d+).+:\d+\s+([\d|a-f|A-F|x|X]+)\]\s*TcrConnection::send:\s*\[([\d|a-f|A-F|x|X]+).*sending
request to endpoint.*bytes:\s*([\d|a-f|A-F]+)"
- )
- match = expression.search(line)
+ match = self.send_trace_expression_base_.search(line)
if match:
parts.append(parser.parse(match.group(1)))
parts.append(match.group(2))
@@ -124,17 +130,16 @@ class ClientMessageDecoder(DecoderBase):
def get_add_security_trace_parts(self, line, parts):
result = False
- expression = re.compile(
- r"(\d\d\d\d\/\d\d\/\d\d
\d\d:\d\d:\d\d\.\d+).*([\d|a-f|A-F|x|X]+)\]\s*TcrMessage::addSecurityPart\s*\[(0x[\d|a-f|A-F]*).*length\s*=\s*(\d+)\s*,\s*encrypted\s+ID\s*=\s*([\d|a-f|A-F]+)"
- )
- match = expression.search(line)
- if match:
- parts.append(parser.parse(match.group(1)))
- parts.append(match.group(2))
- parts.append(match.group(3))
- parts.append(match.group(4))
- parts.append(match.group(5))
- result = True
+
+ if "addSec" in line:
+ match = self.security_trace_expression_.search(line)
+ if match:
+ parts.append(parser.parse(match.group(1)))
+ parts.append(match.group(2))
+ parts.append(match.group(3))
+ parts.append(match.group(4))
+ parts.append(match.group(5))
+ result = True
return result
@@ -191,12 +196,18 @@ class ClientMessageDecoder(DecoderBase):
def request_requires_security_footer(self, message_type):
return message_type in self.message_requires_security_part
+ def is_candidate_line(self, line):
+ return "TcrMess" in line or "TcrConn" in line
+
def process_line(self, line):
connection = None
is_send_trace = False
is_add_security_trace = False
send_trace = {}
+ if not self.is_candidate_line(line):
+ return
+
parts = []
if self.get_send_trace_parts(line, parts):
(
diff --git a/tools/gnmsg/server_message_decoder.py
b/tools/gnmsg/server_message_decoder.py
index 22e9265..e473b0a 100644
--- a/tools/gnmsg/server_message_decoder.py
+++ b/tools/gnmsg/server_message_decoder.py
@@ -50,12 +50,56 @@ class ServerMessageDecoder(DecoderBase):
self.chunk_decoder = ChunkedResponseDecoder()
self.threads_connections_ = {}
- def associate_connection_to_tid(self, line):
- result = False
- expression = re.compile(
+ self.connection_to_tid_expression_ = re.compile(
r"(\d\d:\d\d:\d\d\.\d+).+:\d+\s+([\d|a-f|A-F|x|X]+)\]\s*TcrConnection::send:\s*\[([\d|a-f|A-F|x|X]+).*sending
request to endpoint.*bytes:\s*([\d|a-f|A-F]+)"
)
- match = expression.search(line)
+
+ self.trace_header_with_pointer_expression_ = re.compile(
+ r"(\d\d\d\d\/\d\d\/\d\d
\d\d:\d\d:\d\d\.\d+).+:\d+\s+([\d|a-f|A-F|x|X]+)\]\s*TcrConnection::readMessage\(([\d|a-f|A-F|x|X]+)\):.*received
header from endpoint.*bytes:\s*([\d|a-f|A-F]+)"
+ )
+
+ self.trace_header_without_pointer_expression_ = re.compile(
+ r"(\d\d\d\d\/\d\d\/\d\d
\d\d:\d\d:\d\d\.\d+).*:\d+\s+([\d|a-f|A-F|x|X]+)\]\s*TcrConnection::readMessage:\s*received
header from endpoint.*bytes:\s*([\d|a-f|A-F]+)"
+ )
+
+ self.trace_header_v911_expression_ = re.compile(
+ r"(\d\d\d\d\/\d\d\/\d\d
\d\d:\d\d:\d\d\.\d+).*:\d+\s+(\d+)\]\s*TcrConnection::readMessage: received
header from endpoint.*bytes:\s*([\d| ]+)"
+ )
+
+ self.receive_trace_body_expression_ = re.compile(
+ ":\d+\s+([\d|a-f|A-F|x|X]+)\]\s*TcrConnection::readMessage:
received message body from endpoint.*bytes:\s*([\d|a-f|A-F]+)"
+ )
+
+ self.security_trace_expression_ = re.compile(
+ r"(\d\d\d\d\/\d\d\/\d\d
\d\d:\d\d:\d\d\.\d+).*:\d+\s+(\d+)\]\s*TcrMessage::addSecurityPart\s*\[(0x[\d|a-f|A-F]*).*length\s*=\s*(\d+)\s*,\s*encrypted\s+ID\s*=\s*([\d|a-f|A-F]+)"
+ )
+
+ self.response_header_with_pointer_expression_ = re.compile(
+ r"(\d\d\d\d\/\d\d\/\d\d
\d\d:\d\d:\d\d\.\d+).*:\d+\s+(\d+)\]\s*TcrConnection::readResponseHeader\(([0-9|a-f|A-F|x]+)\):\s*received
header from endpoint\s*([\w|:|\d|\.|-]+);\s*bytes:\s*([\d|a-f|A-F]+)"
+ )
+
+ self.response_header_without_pointer_expression_ = re.compile(
+ r"(\d\d\d\d\/\d\d\/\d\d
\d\d:\d\d:\d\d\.\d+).*:\d+\s+(\d+)\]\s*TcrConnection::readResponseHeader:\s*received
header from endpoint\s*([\w|:|\d|\.|-]+);\s*bytes:\s*([\d|a-f|A-F]+)"
+ )
+
+ self.chunk_header_with_pointer_expression_ = re.compile(
+ r"(\d\d\d\d\/\d\d\/\d\d
\d\d:\d\d:\d\d\.\d+).*:\d+\s+(\d+)\]\s*TcrConnection::readChunkHeader\(([0-9|a-f|A-F|x]+)\):\s*.*,
chunkLen=(\d+), lastChunkAndSecurityFlags=([0-9|a-f|A-F|x]+)"
+ )
+
+ self.chunk_header_without_pointer_expression_ = re.compile(
+ r"(\d\d\d\d\/\d\d\/\d\d
\d\d:\d\d:\d\d\.\d+).*:\d+\s+(\d+)\]\s*TcrConnection::readChunkHeader:\s*.*,
chunkLen=(\d+), lastChunkAndSecurityFlags=([0-9|a-f|A-F|x]+)"
+ )
+
+ self.chunk_bytes_with_pointer_expression_ = re.compile(
+
r"(\d\d:\d\d:\d\d\.\d+).*:\d+\s+(\d+)\]\s*TcrConnection::readChunkBody\(([0-9|a-f|A-F|x]+)\):
\s*received chunk body from
endpoint\s*([\w|:|\d|\.|-]+);\s*bytes:\s*([\d|a-f|A-F]+)"
+ )
+ self.chunk_bytes_without_pointer_expression_ = re.compile(
+ r"(\d\d\d\d\/\d\d\/\d\d
\d\d:\d\d:\d\d\.\d+).*:\d+\s+(\d+)\]\s*TcrConnection::readChunkBody:
\s*received chunk body from
endpoint\s*([\w|:|\d|\.|-]+);\s*bytes:\s*([\d|a-f|A-F]+)"
+ )
+
+ def associate_connection_to_tid(self, line):
+ result = False
+ match = self.connection_to_tid_expression_.search(line)
if match:
tid = match.group(2)
connection = match.group(3)
@@ -66,10 +110,7 @@ class ServerMessageDecoder(DecoderBase):
def get_receive_trace_header_with_pointer(self, line, parts):
result = False
- expression = re.compile(
- r"(\d\d\d\d\/\d\d\/\d\d
\d\d:\d\d:\d\d\.\d+).+:\d+\s+([\d|a-f|A-F|x|X]+)\]\s*TcrConnection::readMessage\(([\d|a-f|A-F|x|X]+)\):.*received
header from endpoint.*bytes:\s*([\d|a-f|A-F]+)"
- )
- match = expression.search(line)
+ match = self.trace_header_with_pointer_expression_.search(line)
if match:
date_time = parser.parse(match.group(1))
tid = match.group(2)
@@ -86,10 +127,7 @@ class ServerMessageDecoder(DecoderBase):
def get_receive_trace_header_without_pointer(self, line, parts):
result = False
- expression = re.compile(
- r"(\d\d\d\d\/\d\d\/\d\d
\d\d:\d\d:\d\d\.\d+).*:\d+\s+([\d|a-f|A-F|x|X]+)\]\s*TcrConnection::readMessage:\s*received
header from endpoint.*bytes:\s*([\d|a-f|A-F]+)"
- )
- match = expression.search(line)
+ match = self.trace_header_without_pointer_expression_.search(line)
if match:
tid = match.group(2)
parts.append(parser.parse(match.group(1)))
@@ -112,10 +150,7 @@ class ServerMessageDecoder(DecoderBase):
def get_receive_trace_header_v911(self, line, parts):
result = False
- expression = re.compile(
- r"(\d\d\d\d\/\d\d\/\d\d
\d\d:\d\d:\d\d\.\d+).*:\d+\s+(\d+)\]\s*TcrConnection::readMessage: received
header from endpoint.*bytes:\s*([\d| ]+)"
- )
- match = expression.search(line)
+ match = self.trace_header_v911_expression_.search(line)
if match:
tid = match.group(2)
parts.append(parser.parse(match.group(1)))
@@ -131,10 +166,7 @@ class ServerMessageDecoder(DecoderBase):
def get_receive_trace_body_parts(self, line, parts):
result = False
- expression = re.compile(
- ":\d+\s+([\d|a-f|A-F|x|X]+)\]\s*TcrConnection::readMessage:
received message body from endpoint.*bytes:\s*([\d|a-f|A-F]+)"
- )
- match = expression.search(line)
+ match = self.receive_trace_body_expression_.search(line)
if match:
tid = match.group(1)
parts.append(tid)
@@ -163,10 +195,7 @@ class ServerMessageDecoder(DecoderBase):
def get_add_security_trace_parts(self, line, parts):
result = False
- expression = re.compile(
- r"(\d\d\d\d\/\d\d\/\d\d
\d\d:\d\d:\d\d\.\d+).*:\d+\s+(\d+)\]\s*TcrMessage::addSecurityPart\s*\[(0x[\d|a-f|A-F]*).*length\s*=\s*(\d+)\s*,\s*encrypted\s+ID\s*=\s*([\d|a-f|A-F]+)"
- )
- match = expression.search(line)
+ match = self.security_trace_expression_.search(line)
if match:
parts.append(parser.parse(match.group(1)))
parts.append(match.group(2))
@@ -181,10 +210,7 @@ class ServerMessageDecoder(DecoderBase):
# Check if this is a header for a chunked message
result = False
- expression = re.compile(
- r"(\d\d\d\d\/\d\d\/\d\d
\d\d:\d\d:\d\d\.\d+).*:\d+\s+(\d+)\]\s*TcrConnection::readResponseHeader\(([0-9|a-f|A-F|x]+)\):\s*received
header from endpoint\s*([\w|:|\d|\.|-]+);\s*bytes:\s*([\d|a-f|A-F]+)"
- )
- match = expression.search(line)
+ match = self.response_header_with_pointer_expression_.search(line)
if match:
tid = match.group(2)
connection = match.group(3)
@@ -197,10 +223,7 @@ class ServerMessageDecoder(DecoderBase):
result = True
if not result:
- expression = re.compile(
- r"(\d\d\d\d\/\d\d\/\d\d
\d\d:\d\d:\d\d\.\d+).*:\d+\s+(\d+)\]\s*TcrConnection::readResponseHeader:\s*received
header from endpoint\s*([\w|:|\d|\.|-]+);\s*bytes:\s*([\d|a-f|A-F]+)"
- )
- match = expression.search(line)
+ match =
self.response_header_without_pointer_expression_.search(line)
if match:
parts.append(parser.parse(match.group(1)))
parts.append(match.group(2))
@@ -214,10 +237,7 @@ class ServerMessageDecoder(DecoderBase):
def get_chunk_header(self, line, parts):
# Check if this is a header for a chunked message
result = False
- expression = re.compile(
- r"(\d\d\d\d\/\d\d\/\d\d
\d\d:\d\d:\d\d\.\d+).*:\d+\s+(\d+)\]\s*TcrConnection::readChunkHeader\(([0-9|a-f|A-F|x]+)\):\s*.*,
chunkLen=(\d+), lastChunkAndSecurityFlags=([0-9|a-f|A-F|x]+)"
- )
- match = expression.search(line)
+ match = self.chunk_header_with_pointer_expression_.search(line)
if match:
tid = match.group(2)
connection = match.group(3)
@@ -230,10 +250,7 @@ class ServerMessageDecoder(DecoderBase):
result = True
if not result:
- expression = re.compile(
- r"(\d\d\d\d\/\d\d\/\d\d
\d\d:\d\d:\d\d\.\d+).*:\d+\s+(\d+)\]\s*TcrConnection::readChunkHeader:\s*.*,
chunkLen=(\d+), lastChunkAndSecurityFlags=([0-9|a-f|A-F|x]+)"
- )
- match = expression.search(line)
+ match = self.chunk_header_without_pointer_expression_.search(line)
if match:
parts.append(parser.parse(match.group(1)))
tid = match.group(2)
@@ -252,10 +269,7 @@ class ServerMessageDecoder(DecoderBase):
# Check if this is a message chunk.
# If it is, add it to the chunked decoder
result = False
- expression = re.compile(
-
r"(\d\d:\d\d:\d\d\.\d+).*:\d+\s+(\d+)\]\s*TcrConnection::readChunkBody\(([0-9|a-f|A-F|x]+)\):
\s*received chunk body from
endpoint\s*([\w|:|\d|\.|-]+);\s*bytes:\s*([\d|a-f|A-F]+)"
- )
- match = expression.search(line)
+ match = self.chunk_bytes_with_pointer_expression_.search(line)
if match:
parts.append(parser.parse(match.group(1)))
parts.append(match.group(2))
@@ -265,10 +279,7 @@ class ServerMessageDecoder(DecoderBase):
result = True
if not result:
- expression = re.compile(
- r"(\d\d\d\d\/\d\d\/\d\d
\d\d:\d\d:\d\d\.\d+).*:\d+\s+(\d+)\]\s*TcrConnection::readChunkBody:
\s*received chunk body from
endpoint\s*([\w|:|\d|\.|-]+);\s*bytes:\s*([\d|a-f|A-F]+)"
- )
- match = expression.search(line)
+ match = self.chunk_bytes_without_pointer_expression_.search(line)
if match:
parts.append(parser.parse(match.group(1)))
tid = match.group(2)
@@ -328,6 +339,9 @@ class ServerMessageDecoder(DecoderBase):
if self.receive_trace_parser_ is not None:
return self.receive_trace_parser_(message_bytes)
+ def is_candidate_line(self, line):
+ return "TcrMess" in line or "TcrConn" in line
+
def process_line(self, line):
connection = None
message_bytes = None
@@ -335,6 +349,9 @@ class ServerMessageDecoder(DecoderBase):
chunk_bytes = None
tid = None
+ if not self.is_candidate_line(line):
+ return
+
parts = []
if self.associate_connection_to_tid(line):
pass
