Hi Bruno,
Here is the 'sed' inlining I mentioned earlier.
While adding the type hints I noticed that some docstrings are missing
or are out of date. Some my fault, some not. :)
I'll work on updating them as I make changes. It is easier to do that
than go out of my way to fix them all at once.
To test this change you can use the coreutils import test case. It
uses the config-h module for lib/vfprintf.c and friends.
Collin
From 639d368ed6378e71a5a5478e04c2422239287ad2 Mon Sep 17 00:00:00 2001
From: Collin Funk <collin.fu...@gmail.com>
Date: Wed, 27 Mar 2024 17:39:58 -0700
Subject: [PATCH] gnulib-tool.py: Inline 'sed' invocations used on library
files.
* pygnulib/GLFileSystem.py (GLFileAssistant.__init__): Update type hints
and docstrings to reflect changes necessary for using re.sub() instead
of 'sed'.
(GLFileAssistant.add_or_update): Use re.sub() instead of invoking 'sed'.
* pygnulib/GLImport.py (GLImport.prepare): Update transformation
variables to reflect changes to GLFileAssistant.
---
ChangeLog | 10 ++++++++++
gnulib-tool.py.TODO | 1 -
pygnulib/GLFileSystem.py | 43 +++++++++++++++++++++-------------------
pygnulib/GLImport.py | 11 +++++-----
4 files changed, 38 insertions(+), 27 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index bc31b14bd2..aa7df63120 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,13 @@
+2024-03-27 Collin Funk <collin.fu...@gmail.com>
+
+ gnulib-tool.py: Inline 'sed' invocations used on library files.
+ * pygnulib/GLFileSystem.py (GLFileAssistant.__init__): Update type hints
+ and docstrings to reflect changes necessary for using re.sub() instead
+ of 'sed'.
+ (GLFileAssistant.add_or_update): Use re.sub() instead of invoking 'sed'.
+ * pygnulib/GLImport.py (GLImport.prepare): Update transformation
+ variables to reflect changes to GLFileAssistant.
+
2024-03-27 Bruno Haible <br...@clisp.org>
obstack: Work around ICE with Oracle cc 12.6 (regr. 2023-12-01).
diff --git a/gnulib-tool.py.TODO b/gnulib-tool.py.TODO
index d7da337f20..48c9e39fa8 100644
--- a/gnulib-tool.py.TODO
+++ b/gnulib-tool.py.TODO
@@ -9,7 +9,6 @@ Optimize:
- os.chdir around subprocess creation -> cwd=... argument instead.
- Inline all 'sed' invocations:
main.py:1387: args = ['sed', '-e', sed_table, tempname]
- GLFileSystem.py:382: args = ['sed', '-e', transformer]
Various other refactorings, as deemed useful.
diff --git a/pygnulib/GLFileSystem.py b/pygnulib/GLFileSystem.py
index 0397af4df3..a155c7f0a0 100644
--- a/pygnulib/GLFileSystem.py
+++ b/pygnulib/GLFileSystem.py
@@ -19,6 +19,7 @@ from __future__ import annotations
# Define global imports
#===============================================================================
import os
+import re
import codecs
import filecmp
import subprocess as sp
@@ -163,8 +164,13 @@ class GLFileSystem(object):
class GLFileAssistant(object):
'''GLFileAssistant is used to help with file processing.'''
- def __init__(self, config: GLConfig, transformers: dict = dict()):
- '''Create GLFileAssistant instance.'''
+ def __init__(self, config: GLConfig, transformers: dict[str, tuple[re.Pattern, str] | None] = {}) -> None:
+ '''Create GLFileAssistant instance.
+
+ config stores information shared between classes.
+ transformers is a dictionary which uses a file category as the key. The
+ value accessed is a tuple containing arguments for re.sub() or None if
+ no transformations are needed.'''
if type(config) is not GLConfig:
raise TypeError('config must be a GLConfig, not %s'
% type(config).__name__)
@@ -173,11 +179,11 @@ class GLFileAssistant(object):
% type(transformers).__name__)
for key in ['lib', 'aux', 'main', 'tests']:
if key not in transformers:
- transformers[key] = 's,x,x,'
+ transformers[key] = None
else: # if key in transformers
value = transformers[key]
- if type(value) is not str:
- raise TypeError('transformers[%s] must be a string, not %s'
+ if type(value) is not tuple and value != None:
+ raise TypeError('transformers[%s] must be a tuple or None, not %s'
% (key, type(value).__name__))
self.original = None
self.rewritten = None
@@ -341,10 +347,10 @@ class GLFileAssistant(object):
xoriginal = substart('tests=lib/', 'lib/', original)
lookedup, tmpflag = self.filesystem.lookup(xoriginal)
tmpfile = self.tmpfilename(rewritten)
- sed_transform_lib_file = self.transformers.get('lib', '')
- sed_transform_build_aux_file = self.transformers.get('aux', '')
- sed_transform_main_lib_file = self.transformers.get('main', '')
- sed_transform_testsrelated_lib_file = self.transformers.get('tests', '')
+ sed_transform_lib_file = self.transformers.get('lib')
+ sed_transform_build_aux_file = self.transformers.get('aux')
+ sed_transform_main_lib_file = self.transformers.get('main')
+ sed_transform_testsrelated_lib_file = self.transformers.get('tests')
try: # Try to copy lookedup file to tmpfile
copyfile(lookedup, tmpfile)
ensure_writable(tmpfile)
@@ -352,7 +358,7 @@ class GLFileAssistant(object):
raise GLError(15, lookedup)
# Don't process binary files with sed.
if not (original.endswith(".class") or original.endswith(".mo")):
- transformer = ''
+ transformer = None
if original.startswith('lib/'):
if sed_transform_main_lib_file:
transformer = sed_transform_main_lib_file
@@ -362,16 +368,13 @@ class GLFileAssistant(object):
elif original.startswith('tests=lib/'):
if sed_transform_testsrelated_lib_file:
transformer = sed_transform_testsrelated_lib_file
- if transformer:
- args = ['sed', '-e', transformer]
- stdin = codecs.open(lookedup, 'rb', 'UTF-8')
- try: # Try to transform file
- data = sp.check_output(args, stdin=stdin, shell=False)
- data = data.decode("UTF-8")
- except Exception as error:
- raise GLError(16, lookedup)
- with codecs.open(tmpfile, 'wb', 'UTF-8') as file:
- file.write(data)
+ if transformer != None:
+ # Read the file that we looked up.
+ with open(lookedup, 'r', encoding='utf-8') as file:
+ src_data = file.read()
+ # Write the transformed data to the temporary file.
+ with open(tmpfile, 'w', encoding='utf-8') as file:
+ file.write(re.sub(transformer[0], transformer[1], src_data))
path = joinpath(self.config['destdir'], rewritten)
if isfile(path):
# The file already exists.
diff --git a/pygnulib/GLImport.py b/pygnulib/GLImport.py
index fcaffea650..9d80108d46 100644
--- a/pygnulib/GLImport.py
+++ b/pygnulib/GLImport.py
@@ -941,19 +941,18 @@ AC_DEFUN([%s_FILE_LIST], [\n''' % macro_prefix
print(notice)
# Determine script to apply to imported library files.
- sed_transform_lib_file = ''
+ sed_transform_lib_file = None
if 'config-h' in [ str(module)
for module in main_modules ]:
- sed_transform_lib_file += '''
- s/^#ifdef[\t ]*HAVE_CONFIG_H[\t ]*$/#if 1/
- '''
+ sed_transform_lib_file = (re.compile(r'^#ifdef[\t ]*HAVE_CONFIG_H[\t ]*$', re.MULTILINE), r'#if 1')
+
sed_transform_main_lib_file = sed_transform_lib_file
# Determine script to apply to auxiliary files that go into $auxdir/.
- sed_transform_build_aux_file = ''
+ sed_transform_build_aux_file = None
# Determine script to apply to library files that go into $testsbase/.
- sed_transform_testsrelated_lib_file = sed_transform_lib_file
+ sed_transform_testsrelated_lib_file = None
# Determine the final file lists.
main_filelist, tests_filelist = \
--
2.44.0
