Re: [OE-core] [PATCH] scripts: add buildstats-summary
On 21 Mar 2023, at 17:55, Khem Raj wrote: > > This looks useful, I tried it out and it worked in one case but failed in > few others with > > % buildstats-summary build/tmp/buildstats/20230319021406 > Traceback (most recent call last): > File "/mnt/b/yoe/master/sources/poky/scripts/buildstats-summary", > line 126, in >sys.exit(main()) > File "/mnt/b/yoe/master/sources/poky/scripts/buildstats-summary", > line 119, in main >bs = read_buildstats(args.buildstats) > File "/mnt/b/yoe/master/sources/poky/scripts/buildstats-summary", > line 54, in read_buildstats >return buildstats.BuildStats.from_dir(path) > File "/mnt/b/yoe/master/sources/poky/scripts/lib/buildstats.py", > line 256, in from_dir >build_started, build_elapsed = buildstats.parse_top_build_stats(top_stats) > File "/mnt/b/yoe/master/sources/poky/scripts/lib/buildstats.py", > line 245, in parse_top_build_stats >return start, elapsed > UnboundLocalError: local variable 'elapsed' referenced before assignment Looks like that build was either in progress or bitbake was killed and it didn’t finish writing the files. I’ll make it default the elapsed time to 0 to handle this case. Ross -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#178984): https://lists.openembedded.org/g/openembedded-core/message/178984 Mute This Topic: https://lists.openembedded.org/mt/97760470/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [PATCH] scripts: add buildstats-summary
here is the link for faulty one
https://uclibc.org/~kraj/20230319021406.tar.xz
On Tue, Mar 21, 2023 at 10:55 AM Khem Raj wrote:
>
> This looks useful, I tried it out and it worked in one case but failed in
> few others with
>
> % buildstats-summary build/tmp/buildstats/20230319021406
> Traceback (most recent call last):
> File "/mnt/b/yoe/master/sources/poky/scripts/buildstats-summary",
> line 126, in
> sys.exit(main())
> File "/mnt/b/yoe/master/sources/poky/scripts/buildstats-summary",
> line 119, in main
> bs = read_buildstats(args.buildstats)
> File "/mnt/b/yoe/master/sources/poky/scripts/buildstats-summary",
> line 54, in read_buildstats
> return buildstats.BuildStats.from_dir(path)
> File "/mnt/b/yoe/master/sources/poky/scripts/lib/buildstats.py",
> line 256, in from_dir
> build_started, build_elapsed = buildstats.parse_top_build_stats(top_stats)
> File "/mnt/b/yoe/master/sources/poky/scripts/lib/buildstats.py",
> line 245, in parse_top_build_stats
> return start, elapsed
> UnboundLocalError: local variable 'elapsed' referenced before assignment
>
> On Tue, Mar 21, 2023 at 10:39 AM Ross Burton wrote:
> >
> > This script will write a summary of the buildstats to the terminal,
> > sorted by start time or duration, optionally hiding short tasks, and
> > highlighting long running tasks.
> >
> > Signed-off-by: Ross Burton
> > ---
> > scripts/buildstats-summary | 126 +
> > 1 file changed, 126 insertions(+)
> > create mode 100755 scripts/buildstats-summary
> >
> > diff --git a/scripts/buildstats-summary b/scripts/buildstats-summary
> > new file mode 100755
> > index 000..89348318afa
> > --- /dev/null
> > +++ b/scripts/buildstats-summary
> > @@ -0,0 +1,126 @@
> > +#! /usr/bin/python3
> > +#
> > +# Dump a summary of the specified buildstats to the terminal, filtering and
> > +# sorting by walltime.
> > +#
> > +# SPDX-License-Identifier: GPL-2.0-only
> > +
> > +import argparse
> > +import dataclasses
> > +import datetime
> > +import enum
> > +import os
> > +import pathlib
> > +import sys
> > +
> > +scripts_path = os.path.dirname(os.path.realpath(__file__))
> > +sys.path.append(os.path.join(scripts_path, "lib"))
> > +import buildstats
> > +
> > +
> > +@dataclasses.dataclass
> > +class Task:
> > +recipe: str
> > +task: str
> > +start: datetime.datetime
> > +duration: datetime.timedelta
> > +
> > +
> > +class Sorting(enum.Enum):
> > +start = 1
> > +duration = 2
> > +
> > +# argparse integration
> > +def __str__(self) -> str:
> > +return self.name
> > +
> > +def __repr__(self) -> str:
> > +return self.name
> > +
> > +@staticmethod
> > +def from_string(s: str):
> > +try:
> > +return Sorting[s]
> > +except KeyError:
> > +return s
> > +
> > +
> > +def read_buildstats(path: pathlib.Path) -> buildstats.BuildStats:
> > +if not path.exists():
> > +raise Exception(f"No such file or directory: {path}")
> > +if path.is_file():
> > +return buildstats.BuildStats.from_file_json(path)
> > +if (path / "build_stats").is_file():
> > +return buildstats.BuildStats.from_dir(path)
> > +raise Exception(f"Cannot find buildstats in {path}")
> > +
> > +
> > +def dump_buildstats(args, bs: buildstats.BuildStats):
> > +tasks = []
> > +for recipe in bs.values():
> > +for task, stats in recipe.tasks.items():
> > +t = Task(
> > +recipe.name,
> > +task,
> > +datetime.datetime.fromtimestamp(stats["start_time"]),
> > +datetime.timedelta(seconds=int(stats.walltime)),
> > +)
> > +tasks.append(t)
> > +
> > +tasks.sort(key=lambda t: getattr(t, args.sort.name))
> > +
> > +minimum = datetime.timedelta(seconds=args.shortest)
> > +highlight = datetime.timedelta(seconds=args.highlight)
> > +
> > +for t in tasks:
> > +if t.duration >= minimum:
> > +line = f"{t.duration}{t.recipe}:{t.task}"
> > +if t.duration >= highlight:
> > +print(f"\033[1m{line}\033[0m")
> > +else:
> > +print(line)
> > +
> > +
> > +def main(argv=None) -> int:
> > +parser = argparse.ArgumentParser(
> > +formatter_class=argparse.ArgumentDefaultsHelpFormatter
> > +)
> > +
> > +parser.add_argument(
> > +"buildstats", metavar="BUILDSTATS", help="Buildstats file",
> > type=pathlib.Path
> > +)
> > +parser.add_argument(
> > +"--sort",
> > +"-s",
> > +type=Sorting.from_string,
> > +choices=list(Sorting),
> > +default=Sorting.start,
> > +help="Sort tasks",
> > +)
> > +parser.add_argument(
> > +"--shortest",
> > +"-t",
> > +type=int,
> > +default=1,
> > +metavar="SECS",
> > +help="Hide tasks shorter than SECS seconds",
> > +)
> >
Re: [OE-core] [PATCH] scripts: add buildstats-summary
This looks useful, I tried it out and it worked in one case but failed in
few others with
% buildstats-summary build/tmp/buildstats/20230319021406
Traceback (most recent call last):
File "/mnt/b/yoe/master/sources/poky/scripts/buildstats-summary",
line 126, in
sys.exit(main())
File "/mnt/b/yoe/master/sources/poky/scripts/buildstats-summary",
line 119, in main
bs = read_buildstats(args.buildstats)
File "/mnt/b/yoe/master/sources/poky/scripts/buildstats-summary",
line 54, in read_buildstats
return buildstats.BuildStats.from_dir(path)
File "/mnt/b/yoe/master/sources/poky/scripts/lib/buildstats.py",
line 256, in from_dir
build_started, build_elapsed = buildstats.parse_top_build_stats(top_stats)
File "/mnt/b/yoe/master/sources/poky/scripts/lib/buildstats.py",
line 245, in parse_top_build_stats
return start, elapsed
UnboundLocalError: local variable 'elapsed' referenced before assignment
On Tue, Mar 21, 2023 at 10:39 AM Ross Burton wrote:
>
> This script will write a summary of the buildstats to the terminal,
> sorted by start time or duration, optionally hiding short tasks, and
> highlighting long running tasks.
>
> Signed-off-by: Ross Burton
> ---
> scripts/buildstats-summary | 126 +
> 1 file changed, 126 insertions(+)
> create mode 100755 scripts/buildstats-summary
>
> diff --git a/scripts/buildstats-summary b/scripts/buildstats-summary
> new file mode 100755
> index 000..89348318afa
> --- /dev/null
> +++ b/scripts/buildstats-summary
> @@ -0,0 +1,126 @@
> +#! /usr/bin/python3
> +#
> +# Dump a summary of the specified buildstats to the terminal, filtering and
> +# sorting by walltime.
> +#
> +# SPDX-License-Identifier: GPL-2.0-only
> +
> +import argparse
> +import dataclasses
> +import datetime
> +import enum
> +import os
> +import pathlib
> +import sys
> +
> +scripts_path = os.path.dirname(os.path.realpath(__file__))
> +sys.path.append(os.path.join(scripts_path, "lib"))
> +import buildstats
> +
> +
> +@dataclasses.dataclass
> +class Task:
> +recipe: str
> +task: str
> +start: datetime.datetime
> +duration: datetime.timedelta
> +
> +
> +class Sorting(enum.Enum):
> +start = 1
> +duration = 2
> +
> +# argparse integration
> +def __str__(self) -> str:
> +return self.name
> +
> +def __repr__(self) -> str:
> +return self.name
> +
> +@staticmethod
> +def from_string(s: str):
> +try:
> +return Sorting[s]
> +except KeyError:
> +return s
> +
> +
> +def read_buildstats(path: pathlib.Path) -> buildstats.BuildStats:
> +if not path.exists():
> +raise Exception(f"No such file or directory: {path}")
> +if path.is_file():
> +return buildstats.BuildStats.from_file_json(path)
> +if (path / "build_stats").is_file():
> +return buildstats.BuildStats.from_dir(path)
> +raise Exception(f"Cannot find buildstats in {path}")
> +
> +
> +def dump_buildstats(args, bs: buildstats.BuildStats):
> +tasks = []
> +for recipe in bs.values():
> +for task, stats in recipe.tasks.items():
> +t = Task(
> +recipe.name,
> +task,
> +datetime.datetime.fromtimestamp(stats["start_time"]),
> +datetime.timedelta(seconds=int(stats.walltime)),
> +)
> +tasks.append(t)
> +
> +tasks.sort(key=lambda t: getattr(t, args.sort.name))
> +
> +minimum = datetime.timedelta(seconds=args.shortest)
> +highlight = datetime.timedelta(seconds=args.highlight)
> +
> +for t in tasks:
> +if t.duration >= minimum:
> +line = f"{t.duration}{t.recipe}:{t.task}"
> +if t.duration >= highlight:
> +print(f"\033[1m{line}\033[0m")
> +else:
> +print(line)
> +
> +
> +def main(argv=None) -> int:
> +parser = argparse.ArgumentParser(
> +formatter_class=argparse.ArgumentDefaultsHelpFormatter
> +)
> +
> +parser.add_argument(
> +"buildstats", metavar="BUILDSTATS", help="Buildstats file",
> type=pathlib.Path
> +)
> +parser.add_argument(
> +"--sort",
> +"-s",
> +type=Sorting.from_string,
> +choices=list(Sorting),
> +default=Sorting.start,
> +help="Sort tasks",
> +)
> +parser.add_argument(
> +"--shortest",
> +"-t",
> +type=int,
> +default=1,
> +metavar="SECS",
> +help="Hide tasks shorter than SECS seconds",
> +)
> +parser.add_argument(
> +"--highlight",
> +"-g",
> +type=int,
> +default=60,
> +metavar="SECS",
> +help="Highlight tasks longer than SECS seconds",
> +)
> +
> +args = parser.parse_args(argv)
> +
> +bs = read_buildstats(args.buildstats)
> +dump_buildstats(args, bs)
> +
> +return 0
> +
> +
> +if __name__ == "__main__":
> +sys.exit(main())
> --
> 2.34.1
[OE-core] [PATCH] scripts: add buildstats-summary
This script will write a summary of the buildstats to the terminal,
sorted by start time or duration, optionally hiding short tasks, and
highlighting long running tasks.
Signed-off-by: Ross Burton
---
scripts/buildstats-summary | 126 +
1 file changed, 126 insertions(+)
create mode 100755 scripts/buildstats-summary
diff --git a/scripts/buildstats-summary b/scripts/buildstats-summary
new file mode 100755
index 000..89348318afa
--- /dev/null
+++ b/scripts/buildstats-summary
@@ -0,0 +1,126 @@
+#! /usr/bin/python3
+#
+# Dump a summary of the specified buildstats to the terminal, filtering and
+# sorting by walltime.
+#
+# SPDX-License-Identifier: GPL-2.0-only
+
+import argparse
+import dataclasses
+import datetime
+import enum
+import os
+import pathlib
+import sys
+
+scripts_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(scripts_path, "lib"))
+import buildstats
+
+
+@dataclasses.dataclass
+class Task:
+recipe: str
+task: str
+start: datetime.datetime
+duration: datetime.timedelta
+
+
+class Sorting(enum.Enum):
+start = 1
+duration = 2
+
+# argparse integration
+def __str__(self) -> str:
+return self.name
+
+def __repr__(self) -> str:
+return self.name
+
+@staticmethod
+def from_string(s: str):
+try:
+return Sorting[s]
+except KeyError:
+return s
+
+
+def read_buildstats(path: pathlib.Path) -> buildstats.BuildStats:
+if not path.exists():
+raise Exception(f"No such file or directory: {path}")
+if path.is_file():
+return buildstats.BuildStats.from_file_json(path)
+if (path / "build_stats").is_file():
+return buildstats.BuildStats.from_dir(path)
+raise Exception(f"Cannot find buildstats in {path}")
+
+
+def dump_buildstats(args, bs: buildstats.BuildStats):
+tasks = []
+for recipe in bs.values():
+for task, stats in recipe.tasks.items():
+t = Task(
+recipe.name,
+task,
+datetime.datetime.fromtimestamp(stats["start_time"]),
+datetime.timedelta(seconds=int(stats.walltime)),
+)
+tasks.append(t)
+
+tasks.sort(key=lambda t: getattr(t, args.sort.name))
+
+minimum = datetime.timedelta(seconds=args.shortest)
+highlight = datetime.timedelta(seconds=args.highlight)
+
+for t in tasks:
+if t.duration >= minimum:
+line = f"{t.duration}{t.recipe}:{t.task}"
+if t.duration >= highlight:
+print(f"\033[1m{line}\033[0m")
+else:
+print(line)
+
+
+def main(argv=None) -> int:
+parser = argparse.ArgumentParser(
+formatter_class=argparse.ArgumentDefaultsHelpFormatter
+)
+
+parser.add_argument(
+"buildstats", metavar="BUILDSTATS", help="Buildstats file",
type=pathlib.Path
+)
+parser.add_argument(
+"--sort",
+"-s",
+type=Sorting.from_string,
+choices=list(Sorting),
+default=Sorting.start,
+help="Sort tasks",
+)
+parser.add_argument(
+"--shortest",
+"-t",
+type=int,
+default=1,
+metavar="SECS",
+help="Hide tasks shorter than SECS seconds",
+)
+parser.add_argument(
+"--highlight",
+"-g",
+type=int,
+default=60,
+metavar="SECS",
+help="Highlight tasks longer than SECS seconds",
+)
+
+args = parser.parse_args(argv)
+
+bs = read_buildstats(args.buildstats)
+dump_buildstats(args, bs)
+
+return 0
+
+
+if __name__ == "__main__":
+sys.exit(main())
--
2.34.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#178892):
https://lists.openembedded.org/g/openembedded-core/message/178892
Mute This Topic: https://lists.openembedded.org/mt/97760470/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
