commit: b368f6aa5e29879df566a89b19471d0b2928f65d
Author: Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Sun Apr 12 01:13:28 2020 +0000
Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Tue Apr 14 18:29:25 2020 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=b368f6aa
catalyst: Use arch data from TOML, not python modules
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
Makefile | 2 +-
catalyst/base/stagebase.py | 133 ++++++++++++----------------------
catalyst/defaults.py | 1 -
doc/make_subarch_table_guidexml.py | 142 ++++++++++---------------------------
setup.py | 1 +
5 files changed, 84 insertions(+), 195 deletions(-)
diff --git a/Makefile b/Makefile
index 6f7eb102..c937df15 100644
--- a/Makefile
+++ b/Makefile
@@ -27,7 +27,7 @@ $(MAN_PAGES): files/%: doc/%.txt doc/asciidoc.conf Makefile
catalyst | files
files/catalyst.1: doc/subarches.generated.txt | files
files/catalyst-spec.5: doc/subarches.generated.txt doc/targets.generated.txt |
files
-doc/subarches.generated.txt: $(wildcard catalyst/arch/*.py)
doc/make_subarch_table_guidexml.py
+doc/subarches.generated.txt: $(wildcard arch/*.toml)
doc/make_subarch_table_guidexml.py
./doc/make_subarch_table_guidexml.py
doc/targets.generated.txt: doc/make_target_table.py $(wildcard
catalyst/targets/*.py)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index d4fafb74..b01e4825 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -1,6 +1,7 @@
import os
import imp
+import toml
import platform
import shutil
import sys
@@ -65,106 +66,62 @@ class StageBase(TargetBase, ClearBase, GenBase):
GenBase.__init__(self, myspec)
ClearBase.__init__(self, myspec)
- # The semantics of subarchmap and machinemap changed a bit in
2.0.3 to
- # work better with vapier's CBUILD stuff. I've removed the
"monolithic"
- # machinemap from this file and split up its contents amongst
the
- # various arch/foo.py files.
- #
- # When register() is called on each module in the arch/ dir, it
now
- # returns a tuple instead of acting on the subarchmap dict that
is
- # passed to it. The tuple contains the values that were
previously
- # added to subarchmap as well as a new list of CHOSTs that go
along
- # with that arch. This allows us to build machinemap on the fly
based
- # on the keys in subarchmap and the values of the 2nd list
returned
- # (tmpmachinemap).
- #
- # Also, after talking with vapier. I have a slightly better
idea of what
- # certain variables are used for and what they should be set
to. Neither
- # 'buildarch' or 'hostarch' are used directly, so their value
doesn't
- # really matter. They are just compared to determine if we are
- # cross-compiling. Because of this, they are just set to the
name of the
- # module in arch/ that the subarch is part of to make things
simpler.
- # The entire build process is still based off of 'subarch' like
it was
- # previously. -agaffney
-
self.makeconf = {}
- self.archmap = {}
- self.subarchmap = {}
- machinemap = {}
- arch_dir = self.settings["archdir"] + "/"
- log.debug("Begin loading arch modules...")
- for x in [
- x[:-3] for x in os.listdir(arch_dir) if
x.endswith(".py")
- and x != "__init__.py"]:
- log.debug("\tLoading %s", x)
- try:
- fh = open(arch_dir + x + ".py")
- # This next line loads the plugin as a module
and
- # assigns it to archmap[x]
- self.archmap[x] = imp.load_module(x, fh,
arch_dir + x + ".py",
- (".py", "r", imp.PY_SOURCE))
- # This next line registers all the subarches
- # supported in the plugin
- tmpsubarchmap, tmpmachinemap =
self.archmap[x].register()
- self.subarchmap.update(tmpsubarchmap)
- for machine in tmpmachinemap:
- machinemap[machine] = x
- for subarch in tmpsubarchmap:
- machinemap[subarch] = x
- fh.close()
- except IOError:
- # This message should probably change a bit,
since everything in
- # the dir should load just fine. If it doesn't,
it's probably a
- # syntax error in the module
- log.warning("Can't find/load %s.py plugin in
%s", x, arch_dir)
- log.debug("Loaded arch module: %s", self.archmap[x])
if "chost" in self.settings:
- hostmachine = self.settings["chost"].split("-")[0]
- if hostmachine not in machinemap:
- raise CatalystError("Unknown host machine type
" + hostmachine)
- self.settings["hostarch"] = machinemap[hostmachine]
+ host = self.settings["chost"].split("-")[0]
else:
- hostmachine = self.settings["subarch"]
- if hostmachine in machinemap:
- hostmachine = machinemap[hostmachine]
- self.settings["hostarch"] = hostmachine
+ host = self.settings["subarch"]
+ self.settings["hostarch"] = host
+
if "cbuild" in self.settings:
- buildmachine = self.settings["cbuild"].split("-")[0]
+ build = self.settings["cbuild"].split("-")[0]
+ else:
+ build = platform.machine()
+ self.settings["buildarch"] = build
+
+ arch_dir = normpath(self.settings['sharedir'] + '/arch/')
+
+ log.debug("Searching arch definitions...")
+ for x in [x for x in os.listdir(arch_dir) if
x.endswith('.toml')]:
+ log.debug("\tTrying %s", x)
+ name = x[:-len('.toml')]
+
+ with open(arch_dir + x) as file:
+ arch_config = toml.load(file)
+
+ # Search for a subarchitecture in each arch in
the arch_config
+ for arch in [x for x in arch_config if
x.startswith(name) and host in arch_config[x]]:
+
self.settings.update(arch_config[arch][host])
+ setarch = arch_config.get('setarch', {})
+ break
+ else:
+ # Didn't find a matching
subarchitecture, keep searching
+ continue
+
+ break
else:
- buildmachine = platform.machine()
- if buildmachine not in machinemap:
- raise CatalystError("Unknown build machine type " +
buildmachine)
- self.settings["buildarch"] = machinemap[buildmachine]
-
- # Call arch constructor, pass our settings
- try:
- self.arch =
self.subarchmap[self.settings["subarch"]](self.settings)
- except KeyError:
- log.critical(
- 'Invalid subarch: %s\n'
- 'Choose one of the following:\n'
- ' %s',
- self.settings['subarch'], '
'.join(self.subarchmap))
-
- if 'setarch_arch' in self.settings and platform.machine() ==
self.settings["setarch_build"]:
- self.settings["CHROOT"] = f'setarch
{self.settings["setarch_arch"]} chroot'
- self.settings["crosscompile"] = False
+ raise CatalystError("Unknown host machine type " + host)
+
+ if setarch.get('if_build', '') == platform.machine():
+ chroot = f'setarch {setarch["arch"]} chroot'
else:
- self.settings["CHROOT"] = 'chroot'
- self.settings["crosscompile"] = \
- self.settings["hostarch"] !=
self.settings["buildarch"]
+ chroot = 'chroot'
+ self.settings["CHROOT"] = chroot
+
+ self.settings["crosscompile"] = \
+ build != host and not chroot.startswith('setarch')
log.notice('Using target: %s', self.settings['target'])
# Print a nice informational message
- if self.settings["buildarch"] == self.settings["hostarch"]:
- log.info('Building natively for %s',
self.settings['hostarch'])
- elif self.settings["crosscompile"]:
+ if self.settings["crosscompile"]:
log.info('Cross-compiling on %s for different machine
type %s',
- self.settings['buildarch'],
self.settings['hostarch'])
- else:
+ build, host)
+ elif chroot.startswith('setarch'):
log.info('Building on %s for alternate personality type
%s',
- self.settings['buildarch'],
self.settings['hostarch'])
+ build, host)
+ else:
+ log.info('Building natively for %s', host)
# This must be set first as other set_ options depend on this
self.set_spec_prefix()
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 349f16f6..60a2ac9c 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -32,7 +32,6 @@ MAINREPO = "gentoo"
PORTDIR = REPODIR + "/" + MAINREPO
confdefaults={
- "archdir": "%(PythonDir)s/arch",
"comp_prog": COMPRESSOR_PROGRAM_OPTIONS['linux'],
"compression_mode": 'lbzip2',
"compressor_arch": None,
diff --git a/doc/make_subarch_table_guidexml.py
b/doc/make_subarch_table_guidexml.py
index 84624dc1..7ee95bde 100755
--- a/doc/make_subarch_table_guidexml.py
+++ b/doc/make_subarch_table_guidexml.py
@@ -1,120 +1,52 @@
#!/usr/bin/env python
-# Copyright (C) 2011 Sebastian Pipping <[email protected]>
-# Copyright (C) 2013 Brian dolbec <[email protected]>
# Licensed under GPL v2 or later
-import os
-import re
+import pathlib
import sys
import textwrap
-
-
-_pattern_arch_generic = re.compile('^class
arch_([a-z0-9_.-]+)\\(generic_([a-z0-9_.-]+)\\):')
-_pattern_arch_arch = re.compile('^class
arch_([a-z0-9_.-]+)\\(arch_([a-z0-9_.-]+)\\):')
-_pattern_title = re.compile('"([a-z0-9_.-]+)"[ \\t]*:[
\\t]*arch_([a-z0-9_.-]+),?')
-
-_pattern_arch_genericliases = {
- 'armeb':'arm',
- 'sheb':'sh',
- 'mipsel':'mips',
- 'mips64el':'mips64',
-}
-
-
-def handle_line(line, subarch_title_to_subarch_id,
subarch_id_to_pattern_arch_genericrch_id):
- x = _pattern_arch_generic.search(line)
- if x is not None:
- subarch = x.group(1)
- arch = x.group(2)
-
- # Apply alias grouping
- arch = _pattern_arch_genericliases.get(arch, arch)
-
- assert subarch not in subarch_id_to_pattern_arch_genericrch_id
- subarch_id_to_pattern_arch_genericrch_id[subarch] = arch
-
- return
-
- x = _pattern_arch_arch.search(line)
- if x is not None:
- child_subarch = x.group(1)
- parent_subarch = x.group(2)
-
- assert child_subarch not in
subarch_id_to_pattern_arch_genericrch_id
- subarch_id_to_pattern_arch_genericrch_id[child_subarch] =
subarch_id_to_pattern_arch_genericrch_id[parent_subarch]
-
- return
-
- for x in re.finditer(_pattern_title, line):
- subarch_title = x.group(1)
- subarch_id = x.group(2)
-
- assert subarch_title not in subarch_title_to_subarch_id
- subarch_title_to_subarch_id[subarch_title] = subarch_id
-
-
-def handle_file(fn, subarch_title_to_subarch_id,
subarch_id_to_pattern_arch_genericrch_id):
- f = open(fn, 'r')
- for l in f:
- line = l.rstrip()
- handle_line(line, subarch_title_to_subarch_id,
subarch_id_to_pattern_arch_genericrch_id)
- f.close()
-
-
-def dump(subarch_title_to_subarch_id,
subarch_id_to_pattern_arch_genericrch_id):
- arch_id_to_subarch_titles = dict()
- for subarch_title, subarch_id in subarch_title_to_subarch_id.items():
- arch_id =
subarch_id_to_pattern_arch_genericrch_id.get(subarch_id, subarch_id)
-
- if arch_id not in arch_id_to_subarch_titles:
- arch_id_to_subarch_titles[arch_id] = set()
- arch_id_to_subarch_titles[arch_id].add(subarch_title)
-
- # GuideXML version
- f = open('doc/subarches.generated.xml', 'w')
- f.write("""
-<table>
-<tr>
-<th>Architecture</th>
-<th>Sub-architectures</th>
-</tr>
-""")
- for arch_id, subarch_titles in
sorted(arch_id_to_subarch_titles.items()):
- f.write("""<tr>
-<ti><c>%s</c></ti>
-<ti><c>%s</c></ti>
-</tr>
-""" % (arch_id, '\n'.join(textwrap.wrap(' '.join(sorted(subarch_titles)),
60))))
-
- f.write("""</table>
-""")
- f.close()
-
- # Asciidoc
- f = open('doc/subarches.generated.txt', 'w')
- for arch_id, subarch_titles in
sorted(arch_id_to_subarch_titles.items()):
- f.write('*%s*::\n' % arch_id)
- f.write(' %s\n' % ', '.join(sorted(subarch_titles)))
- f.write('\n')
- f.close()
+import toml
+
+
+def write_guidexml(arch_to_subarch):
+ with open('doc/subarches.generated.xml', 'w') as f:
+ f.write(textwrap.dedent("""\
+ <table>
+ <tr>
+ <th>Architecture</th>
+ <th>Sub-architectures</th>
+ </tr>
+ """))
+ for arch, subarches in sorted(arch_to_subarch.items()):
+ f.write(textwrap.dedent("""\
+ <tr>
+ <ti><c>%s</c></ti>
+ <ti><c>%s</c></ti>
+ </tr>
+ """) % (arch, '\n'.join(textwrap.wrap('
'.join(sorted(subarches)), 60))))
+ f.write("</table>\n")
+
+
+def write_asciidoc(arch_to_subarch):
+ with open('doc/subarches.generated.txt', 'w') as f:
+ for arch, subarches in sorted(arch_to_subarch.items()):
+ f.write('*%s*::\n' % arch)
+ f.write(' %s\n' % ', '.join(sorted(subarches)))
+ f.write('\n')
def main(_argv):
- subarch_title_to_subarch_id = dict()
- subarch_id_to_pattern_arch_genericrch_id = dict()
+ arch_to_subarch = {}
+ p = pathlib.Path('arch')
- for dirpath, _dirnames, filenames in os.walk('catalyst/arch'):
- for _fn in filenames:
- if not _fn.endswith('.py'):
- continue
- if _fn == '__init__.py':
- continue
+ for file in p.glob('*.toml'):
+ data = toml.load(file)
- fn = os.path.join(dirpath, _fn)
- handle_file(fn, subarch_title_to_subarch_id,
subarch_id_to_pattern_arch_genericrch_id)
+ for arch in [x for x in data if x != 'setarch']:
+ arch_to_subarch.update({arch: list(data[arch].keys())})
- dump(subarch_title_to_subarch_id,
subarch_id_to_pattern_arch_genericrch_id)
+ write_guidexml(arch_to_subarch)
+ write_asciidoc(arch_to_subarch)
if __name__ == '__main__':
diff --git a/setup.py b/setup.py
index 060c886f..dba6cd6c 100755
--- a/setup.py
+++ b/setup.py
@@ -48,6 +48,7 @@ _data_files = [('/etc/catalyst',
['etc/catalyst.conf','etc/catalystrc']),
('/usr/share/man/man1', ['files/catalyst.1']),
('/usr/share/man/man5', ['files/catalyst-config.5',
'files/catalyst-spec.5'])
]
+_data_files.extend(_files('share/catalyst/arch', 'arch'))
_data_files.extend(_files('share/catalyst/livecd', 'livecd'))
_data_files.extend(_files('share/catalyst/targets', 'targets'))
