This is an automated email from the ASF dual-hosted git repository. tanxinyu pushed a commit to branch e2e_parser in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit db8b616384435952e533915517bda232412c195d Author: LebronAl <[email protected]> AuthorDate: Mon Nov 8 21:23:00 2021 +0800 add tools --- testcontainer/src/tool/README.md | 33 +++++++++++++++++++ testcontainer/src/tool/parser.py | 68 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+) diff --git a/testcontainer/src/tool/README.md b/testcontainer/src/tool/README.md new file mode 100644 index 0000000..3b943af --- /dev/null +++ b/testcontainer/src/tool/README.md @@ -0,0 +1,33 @@ +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> + +## Motivation +The current E2E framework writes all process logs into a single test log. This approach makes it difficult to find the root cause when there are failed tests with unstable recurrence. So we need a tool that can find the failed tests in this log and separate the logs from the different nodes. + +## Usage +1. Download log archive from CI. + +2. Parse log. +``` +python3 parser.py [filename] +``` + +3. View the separated logs in the current directory. \ No newline at end of file diff --git a/testcontainer/src/tool/parser.py b/testcontainer/src/tool/parser.py new file mode 100644 index 0000000..2f68e42 --- /dev/null +++ b/testcontainer/src/tool/parser.py @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import os +import sys +import re + +pattern = re.compile(r'docker\-java\-stream\-+(\d+)') + + +def getAllLogs(filename): + with open(filename, "r") as f: + data = f.read() + return data.split("up -d")[1:] + + +def writeAllLogs(filename, content): + with open(filename, "w") as f: + for row in content: + f.write(row) + f.write("\n") + + +def getNodes(log): + ids = pattern.findall(log) + nodes = {} + for id in ids: + if not nodes.__contains__(id): + nodes[id] = [] + return nodes + + +if __name__ == "__main__": + logs = getAllLogs(sys.argv[1]) + count = 0 + for i in range(len(logs)): + if logs[i].__contains__("FAILURE!"): + nodes = getNodes(logs[i]) + rows = logs[i].split("\n") + for row in rows: + for id, content in nodes.items(): + if row.__contains__(id): + content.append(row) + if row.__contains__("[ERROR]"): + for content in nodes.values(): + content.append(row) + count = count + 1 + for key, content in nodes.items(): + if not os.path.exists("./{}".format(i)): + os.mkdir("./{}".format(i)) + writeAllLogs("./{}/{}_{}.txt".format(i, i, key), content) + + print("find {} failed tests".format(count))
