Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package python-bincopy for openSUSE:Factory checked in at 2023-12-03 20:48:51 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-bincopy (Old) and /work/SRC/openSUSE:Factory/.python-bincopy.new.25432 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-bincopy" Sun Dec 3 20:48:51 2023 rev:2 rq:1130492 version:20.0.0 Changes: -------- --- /work/SRC/openSUSE:Factory/python-bincopy/python-bincopy.changes 2023-06-28 21:33:56.222064560 +0200 +++ /work/SRC/openSUSE:Factory/.python-bincopy.new.25432/python-bincopy.changes 2023-12-03 20:49:12.770656631 +0100 @@ -1,0 +2,11 @@ +Sat Dec 2 16:47:49 UTC 2023 - Dirk Müller <[email protected]> + +- update to 20.0.0: + * Fix chunk merging for work size > 1 + * Simply chunk overlap check + * Fixup: consider word size when merging chunks + * Add tests for padding + * Make chunk padding word sized + * fix elf + +------------------------------------------------------------------- Old: ---- bincopy-17.14.5.tar.gz New: ---- bincopy-20.0.0.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-bincopy.spec ++++++ --- /var/tmp/diff_new_pack.ijD908/_old 2023-12-03 20:49:13.486682932 +0100 +++ /var/tmp/diff_new_pack.ijD908/_new 2023-12-03 20:49:13.486682932 +0100 @@ -18,28 +18,28 @@ %{?sle15_python_module_pythons} Name: python-bincopy -Version: 17.14.5 +Version: 20.0.0 Release: 0 Summary: Mangling of various file formats that conveys binary information License: MIT URL: https://github.com/eerimoq/bincopy Source: https://files.pythonhosted.org/packages/source/b/bincopy/bincopy-%{version}.tar.gz -BuildRequires: python-rpm-macros BuildRequires: %{python_module pip} BuildRequires: %{python_module setuptools} BuildRequires: %{python_module wheel} +BuildRequires: python-rpm-macros # SECTION test requirements BuildRequires: %{python_module pytest} BuildRequires: %{python_module argparse_addons} -BuildRequires: %{python_module pyelftools} BuildRequires: %{python_module humanfriendly} +BuildRequires: %{python_module pyelftools} # /SECTION Requires: python-argparse_addons +Requires: fdupes Requires: python-humanfriendly Requires: python-pyelftools -Requires: fdupes Requires(post): update-alternatives -Requires(postun): update-alternatives +Requires(postun):update-alternatives BuildArch: noarch %python_subpackages ++++++ bincopy-17.14.5.tar.gz -> bincopy-20.0.0.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/bincopy-17.14.5/PKG-INFO new/bincopy-20.0.0/PKG-INFO --- old/bincopy-17.14.5/PKG-INFO 2023-02-16 18:49:08.069267300 +0100 +++ new/bincopy-20.0.0/PKG-INFO 2023-10-26 21:13:10.971764600 +0200 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: bincopy -Version: 17.14.5 +Version: 20.0.0 Summary: Mangling of various file formats that conveys binary information (Motorola S-Record, Intel HEX and binary files). Home-page: https://github.com/eerimoq/bincopy Author: Erik Moqvist @@ -14,6 +14,9 @@ Classifier: Programming Language :: Python :: 3.9 Requires-Python: >=3.6 License-File: LICENSE +Requires-Dist: humanfriendly +Requires-Dist: argparse_addons>=0.4.0 +Requires-Dist: pyelftools About ===== @@ -64,6 +67,10 @@ 3F 01 56 70 2B 5E 71 2B 72 2B 73 21 46 01 34 21 q + >>> print(f.as_verilog_vmem()) + @00000100 21 46 01 36 01 21 47 01 36 00 7E FE 09 D2 19 01 21 46 01 7E 17 C2 00 01 FF 5F 16 00 21 48 01 19 + @00000120 19 4E 79 23 46 23 96 57 78 23 9E DA 3F 01 B2 CA 3F 01 56 70 2B 5E 71 2B 72 2B 73 21 46 01 34 21 + >>> f.as_binary() bytearray(b'!F\x016\x01!G\x016\x00~\xfe\t\xd2\x19\x01!F\x01~\x17\xc2\x00\x01 \xff_\x16\x00!H\x01\x19\x19Ny#F#\x96Wx#\x9e\xda?\x01\xb2\xca?\x01Vp+^q+r+s! diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/bincopy-17.14.5/README.rst new/bincopy-20.0.0/README.rst --- old/bincopy-17.14.5/README.rst 2023-02-16 18:48:54.000000000 +0100 +++ new/bincopy-20.0.0/README.rst 2023-10-26 21:12:57.000000000 +0200 @@ -47,6 +47,10 @@ 3F 01 56 70 2B 5E 71 2B 72 2B 73 21 46 01 34 21 q + >>> print(f.as_verilog_vmem()) + @00000100 21 46 01 36 01 21 47 01 36 00 7E FE 09 D2 19 01 21 46 01 7E 17 C2 00 01 FF 5F 16 00 21 48 01 19 + @00000120 19 4E 79 23 46 23 96 57 78 23 9E DA 3F 01 B2 CA 3F 01 56 70 2B 5E 71 2B 72 2B 73 21 46 01 34 21 + >>> f.as_binary() bytearray(b'!F\x016\x01!G\x016\x00~\xfe\t\xd2\x19\x01!F\x01~\x17\xc2\x00\x01 \xff_\x16\x00!H\x01\x19\x19Ny#F#\x96Wx#\x9e\xda?\x01\xb2\xca?\x01Vp+^q+r+s! diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/bincopy-17.14.5/bincopy.egg-info/PKG-INFO new/bincopy-20.0.0/bincopy.egg-info/PKG-INFO --- old/bincopy-17.14.5/bincopy.egg-info/PKG-INFO 2023-02-16 18:49:08.000000000 +0100 +++ new/bincopy-20.0.0/bincopy.egg-info/PKG-INFO 2023-10-26 21:13:10.000000000 +0200 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: bincopy -Version: 17.14.5 +Version: 20.0.0 Summary: Mangling of various file formats that conveys binary information (Motorola S-Record, Intel HEX and binary files). Home-page: https://github.com/eerimoq/bincopy Author: Erik Moqvist @@ -14,6 +14,9 @@ Classifier: Programming Language :: Python :: 3.9 Requires-Python: >=3.6 License-File: LICENSE +Requires-Dist: humanfriendly +Requires-Dist: argparse_addons>=0.4.0 +Requires-Dist: pyelftools About ===== @@ -64,6 +67,10 @@ 3F 01 56 70 2B 5E 71 2B 72 2B 73 21 46 01 34 21 q + >>> print(f.as_verilog_vmem()) + @00000100 21 46 01 36 01 21 47 01 36 00 7E FE 09 D2 19 01 21 46 01 7E 17 C2 00 01 FF 5F 16 00 21 48 01 19 + @00000120 19 4E 79 23 46 23 96 57 78 23 9E DA 3F 01 B2 CA 3F 01 56 70 2B 5E 71 2B 72 2B 73 21 46 01 34 21 + >>> f.as_binary() bytearray(b'!F\x016\x01!G\x016\x00~\xfe\t\xd2\x19\x01!F\x01~\x17\xc2\x00\x01 \xff_\x16\x00!H\x01\x19\x19Ny#F#\x96Wx#\x9e\xda?\x01\xb2\xca?\x01Vp+^q+r+s! diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/bincopy-17.14.5/bincopy.py new/bincopy-20.0.0/bincopy.py --- old/bincopy-17.14.5/bincopy.py 2023-02-16 18:48:54.000000000 +0100 +++ new/bincopy-20.0.0/bincopy.py 2023-10-26 21:12:57.000000000 +0200 @@ -20,7 +20,7 @@ __author__ = 'Erik Moqvist' -__version__ = '17.14.5' +__version__ = '20.0.0' DEFAULT_WORD_SIZE_BITS = 8 @@ -346,54 +346,72 @@ return True -class _Segment: +class Segment: """A segment is a chunk data with given minimum and maximum address. """ - _Chunk = namedtuple('Chunk', ['address', 'data']) - def __init__(self, minimum_address, maximum_address, data, word_size_bytes): self.minimum_address = minimum_address self.maximum_address = maximum_address self.data = data - self._word_size_bytes = word_size_bytes + self.word_size_bytes = word_size_bytes @property def address(self): - return self.minimum_address // self._word_size_bytes + return self.minimum_address // self.word_size_bytes + + def chunks(self, size=32, alignment=1, padding=b''): + """Yield data chunks of `size` words, aligned as given by `alignment`. + + Each chunk is itself a Segment. + + `size` and `alignment` are in words. `size` must be a multiple of + `alignment`. If set, `padding` must be a word value. - def chunks(self, size=32, alignment=1): - """Return chunks of the data aligned as given by `alignment`. `size` - must be a multiple of `alignment`. Each chunk is returned as a - named two-tuple of its address and data. Both `size` and - `alignment` are in words. + If `padding` is set, the first and final chunks are padded so that: + 1. The first chunk is aligned even if the segment itself is not. + 2. The final chunk's size is a multiple of `alignment`. """ if (size % alignment) != 0: raise Error(f'size {size} is not a multiple of alignment {alignment}') - size *= self._word_size_bytes - alignment *= self._word_size_bytes + if padding and len(padding) != self.word_size_bytes: + raise Error(f'padding must be a word value (size {self.word_size_bytes}),' + f' got {padding}') + + size *= self.word_size_bytes + alignment *= self.word_size_bytes address = self.minimum_address data = self.data - # First chunk may be shorter than `size` due to alignment. + # Apply padding to first and final chunk, if padding is non-empty. + align_offset = address % alignment + address -= align_offset * bool(padding) + data = align_offset // self.word_size_bytes * padding + data + data += (alignment - len(data)) % alignment // self.word_size_bytes * padding + + # First chunk may be non-aligned and shorter than `size` if padding is empty. chunk_offset = (address % alignment) if chunk_offset != 0: first_chunk_size = (alignment - chunk_offset) - yield self._Chunk(address // self._word_size_bytes, - data[:first_chunk_size]) - address += (first_chunk_size // self._word_size_bytes) + yield Segment(address, + address + size, + data[:first_chunk_size], + self.word_size_bytes) + address += first_chunk_size data = data[first_chunk_size:] else: first_chunk_size = 0 for offset in range(0, len(data), size): - yield self._Chunk((address + offset) // self._word_size_bytes, - data[offset:offset + size]) + yield Segment(address + offset, + address + offset + size, + data[offset:offset + size], + self.word_size_bytes) def add_data(self, minimum_address, maximum_address, data, overwrite): """Add given data to this segment. The added data must be adjacent to @@ -464,10 +482,10 @@ self.maximum_address = self.minimum_address + part1_size self.data = part1_data - return _Segment(maximum_address, - maximum_address + len(part2_data), - part2_data, - self._word_size_bytes) + return Segment(maximum_address, + maximum_address + len(part2_data), + part2_data, + self.word_size_bytes) else: # Update this segment. if len(part1_data) > 0: @@ -483,11 +501,11 @@ def __eq__(self, other): if isinstance(other, tuple): return self.address, self.data == other - elif isinstance(other, _Segment): + elif isinstance(other, Segment): return ((self.minimum_address == other.minimum_address) and (self.maximum_address == other.maximum_address) and (self.data == other.data) - and (self._word_size_bytes == other._word_size_bytes)) + and (self.word_size_bytes == other.word_size_bytes)) else: return False @@ -499,14 +517,20 @@ def __repr__(self): return f'Segment(address={self.address}, data={self.data})' + def __len__(self): + return len(self.data) // self.word_size_bytes + -class _Segments: +_Segment = Segment + + +class Segments: """A list of segments. """ def __init__(self, word_size_bytes): - self._word_size_bytes = word_size_bytes + self.word_size_bytes = word_size_bytes self._current_segment = None self._current_segment_index = None self._list = [] @@ -625,22 +649,51 @@ self._list = new_list - def chunks(self, size=32, alignment=1): - """Iterate over all segments and return chunks of the data aligned as - given by `alignment`. `size` must be a multiple of - `alignment`. Each chunk is returned as a named two-tuple of - its address and data. Both `size` and `alignment` are in - words. + def chunks(self, size=32, alignment=1, padding=b''): + """Iterate over all segments and yield chunks of the data. + + The chunks are `size` words long, aligned as given by `alignment`. + + Each chunk is itself a Segment. + + `size` and `alignment` are in words. `size` must be a multiple of + `alignment`. If set, `padding` must be a word value. + + If `padding` is set, the first and final chunks of each segment are + padded so that: + 1. The first chunk is aligned even if the segment itself is not. + 2. The final chunk's size is a multiple of `alignment`. """ if (size % alignment) != 0: raise Error(f'size {size} is not a multiple of alignment {alignment}') + if padding and len(padding) != self.word_size_bytes: + raise Error(f'padding must be a word value (size {self.word_size_bytes}),' + f' got {padding}') + + previous = Segment(-1, -1, b'', 1) + for segment in self: - for chunk in segment.chunks(size, alignment): + for chunk in segment.chunks(size, alignment, padding): + # When chunks are padded to alignment, the final chunk of the previous + # segment and the first chunk of the current segment may overlap by + # one alignment block. To avoid overwriting data from the lower + # segment, the chunks must be merged. + if chunk.address < previous.address + len(previous): + low = previous.data[-alignment * self.word_size_bytes:] + high = chunk.data[:alignment * self.word_size_bytes] + merged = int.to_bytes(int.from_bytes(low, 'big') ^ + int.from_bytes(high, 'big') ^ + int.from_bytes(alignment * padding, 'big'), + alignment * self.word_size_bytes, 'big') + chunk.data = merged + chunk.data[alignment * self.word_size_bytes:] + yield chunk + previous = chunk + def __len__(self): """Get the number of segments. @@ -649,6 +702,9 @@ return len(self._list) +_Segments = Segments + + class BinFile: """A binary file. @@ -682,7 +738,7 @@ self._header_encoding = header_encoding self._header = None self._execution_start_address = None - self._segments = _Segments(self.word_size_bytes) + self._segments = Segments(self.word_size_bytes) if filenames is not None: if isinstance(filenames, str): @@ -832,10 +888,10 @@ >>> for chunk in binfile.segments.chunks(2): ... print(chunk) ... - Chunk(address=0, data=bytearray(b'\\x00\\x01')) - Chunk(address=2, data=bytearray(b'\\x02')) - Chunk(address=10, data=bytearray(b'\\x03\\x04')) - Chunk(address=12, data=bytearray(b'\\x05')) + Segment(address=0, data=bytearray(b'\\x00\\x01')) + Segment(address=2, data=bytearray(b'\\x02')) + Segment(address=10, data=bytearray(b'\\x03\\x04')) + Segment(address=12, data=bytearray(b'\\x05')) Each segment can be split into smaller pieces using the `chunks(size=32, alignment=1)` method on a single segment. @@ -846,11 +902,11 @@ ... print(chunk) ... Segment(address=0, data=bytearray(b'\\x00\\x01\\x02')) - Chunk(address=0, data=bytearray(b'\\x00\\x01')) - Chunk(address=2, data=bytearray(b'\\x02')) + Segment(address=0, data=bytearray(b'\\x00\\x01')) + Segment(address=2, data=bytearray(b'\\x02')) Segment(address=10, data=bytearray(b'\\x03\\x04\\x05')) - Chunk(address=10, data=bytearray(b'\\x03\\x04')) - Chunk(address=12, data=bytearray(b'\\x05')) + Segment(address=10, data=bytearray(b'\\x03\\x04')) + Segment(address=12, data=bytearray(b'\\x05')) """ @@ -893,10 +949,10 @@ self._header = data elif type_ in '123': address *= self.word_size_bytes - self._segments.add(_Segment(address, - address + size, - data, - self.word_size_bytes), + self._segments.add(Segment(address, + address + size, + data, + self.word_size_bytes), overwrite) elif type_ in '789': self.execution_start_address = address @@ -924,10 +980,10 @@ + extended_segment_address + extended_linear_address) address *= self.word_size_bytes - self._segments.add(_Segment(address, - address + size, - data, - self.word_size_bytes), + self._segments.add(Segment(address, + address + size, + data, + self.word_size_bytes), overwrite) elif type_ == IHEX_END_OF_FILE: pass @@ -987,10 +1043,10 @@ if address is None: raise Error("missing section address") - self._segments.add(_Segment(address, - address + size, - data, - self.word_size_bytes), + self._segments.add(Segment(address, + address + size, + data, + self.word_size_bytes), overwrite) if size == TI_TXT_BYTES_PER_LINE: @@ -1025,10 +1081,10 @@ for word in words: if word.startswith('@'): if address is not None: - self._segments.add(_Segment(address, - address + len(chunk), - chunk, - self.word_size_bytes)) + self._segments.add(Segment(address, + address + len(chunk), + chunk, + self.word_size_bytes)) address = int(word[1:], 16) * word_size_bytes chunk = b'' @@ -1036,10 +1092,10 @@ chunk += bytes.fromhex(word) if address is not None and chunk: - self._segments.add(_Segment(address, - address + len(chunk), - chunk, - self.word_size_bytes)) + self._segments.add(Segment(address, + address + len(chunk), + chunk, + self.word_size_bytes)) def add_binary(self, data, address=0, overwrite=False): """Add given data at given address. Set `overwrite` to ``True`` to @@ -1048,10 +1104,10 @@ """ address *= self.word_size_bytes - self._segments.add(_Segment(address, - address + len(data), - bytearray(data), - self.word_size_bytes), + self._segments.add(Segment(address, + address + len(data), + bytearray(data), + self.word_size_bytes), overwrite) def add_elf(self, data, overwrite=True): @@ -1086,12 +1142,42 @@ if (section['sh_flags'] & SH_FLAGS.SHF_ALLOC) == 0: continue - self._segments.add(_Segment(address, - address + size, - data[offset:offset + size], - self.word_size_bytes), + self._segments.add(Segment(address, + address + size, + data[offset:offset + size], + self.word_size_bytes), overwrite) + def add_microchip_hex(self, records, overwrite=False): + """Add given Microchip HEX data. + + Microchip's HEX format is identical to Intel's except an address in + the HEX file is twice the actual machine address. For example: + + :02000E00E4C943 + + : Start code + 02 Record contains two data bytes + 000E Address 0x000E; Machine address is 0x000E // 2 == 0x0007 + 00 Record type is data + E4 Low byte at address 0x0007 is 0xE4 + C9 High byte at address 0x0007 is 0xC9 + + Microchip HEX records therefore need to be parsed as if the word size + is one byte, but the parsed data must be handled as if the word size + is two bytes. This is true for both 8-bit PICs such as PIC18 and + 16-bit PICs such as PIC24. + + """ + + self.word_size_bytes = 1 + self.add_ihex(records, overwrite) + self.word_size_bytes = 2 + self.segments.word_size_bytes = 2 + + for segment in self.segments: + segment.word_size_bytes = 2 + def add_file(self, filename, overwrite=False): """Open given file and add its data by guessing its format. The format must be Motorola S-Records, Intel HEX, TI-TXT. Set `overwrite` @@ -1162,6 +1248,15 @@ with open(filename, 'rb') as fin: self.add_elf(fin.read(), overwrite) + def add_microchip_hex_file(self, filename, overwrite=False): + """Open given Microchip HEX file and add its contents. Set `overwrite` + to ``True`` to allow already added data to be overwritten. + + """ + + with open(filename, 'r') as fin: + self.add_microchip_hex(fin.read(), overwrite) + def as_srec(self, number_of_data_bytes=32, address_length_bits=32): """Format the binary file as Motorola S-Records records and return them as a string. @@ -1610,7 +1705,7 @@ fill_size_words = fill_size // self.word_size_bytes if max_words is None or fill_size_words <= max_words: - fill_segments.append(_Segment( + fill_segments.append(Segment( previous_segment_maximum_address, previous_segment_maximum_address + fill_size, value * fill_size_words, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/bincopy-17.14.5/tests/test_bincopy.py new/bincopy-20.0.0/tests/test_bincopy.py --- old/bincopy-17.14.5/tests/test_bincopy.py 2023-02-16 18:48:54.000000000 +0100 +++ new/bincopy-20.0.0/tests/test_bincopy.py 2023-10-26 21:12:57.000000000 +0200 @@ -910,6 +910,12 @@ self.assertEqual(str(cm.exception), 'size 4 is not a multiple of alignment 8') + with self.assertRaises(bincopy.Error) as cm: + list(binfile.segments.chunks(padding=b'\xff\xff')) + + self.assertEqual(str(cm.exception), + r"padding must be a word value (size 1), got b'\xff\xff'") + def test_segment(self): binfile = bincopy.BinFile() binfile.add_binary(b'\x00\x01\x02\x03\x04', 2) @@ -1841,6 +1847,66 @@ with open('tests/files/empty_main.bin', 'rb') as fin: self.assertEqual(binfile.as_binary(padding=b'\x00'), fin.read()) + def test_segment_len(self): + length = 0x100 + word_size_bytes = 1 + segment = bincopy.Segment(0, length, bytes(length), word_size_bytes) + self.assertEqual(length, len(segment)) + + def test_segment_len_16(self): + length = 0x100 + word_size_bytes = 2 + segment = bincopy.Segment(0, + length, + bytes(length * word_size_bytes), + word_size_bytes) + self.assertEqual(length, len(segment)) + + def test_add_microchip_hex_record(self): + binfile = bincopy.BinFile() + binfile.add_microchip_hex(':02000E00E4C943') + self.assertEqual(0x0007, binfile.minimum_address) + first_word = int.from_bytes(binfile[:binfile.minimum_address + 1], 'little') + self.assertEqual(0xC9E4, first_word) + + def test_chunk_padding(self): + records = (':02000004000AF0\n' + ':10B8440000000000000000009630000007770000B0\n') + hexfile = bincopy.BinFile() + hexfile.add_ihex(records) + align = 8 + size = 16 + chunks = hexfile.segments.chunks(size=size, alignment=align, padding=b'\xff') + chunks = list(chunks) + assert not any(c.address % align for c in chunks) + assert not any(len(c) % align for c in chunks) + + def test_merge_chunks(self): + records = (':0A0000001010101010101010101056\n' + ':0A000E001010101010101010101048\n') + hexfile = bincopy.BinFile() + hexfile.add_ihex(records) + align = 8 + size = 16 + chunks = hexfile.segments.chunks(size=size, alignment=align, padding=b'\xff') + chunks = list(chunks) + assert list(chunks[-1]) == [8, b'\x10\x10\xff\xff\xff\xff\x10\x10\x10\x10\x10' + b'\x10\x10\x10\x10\x10'] + + def test_merge_chunks_16(self): + records = (':1000000010101010101010101010101010101010F0\n' + ':10000A0010101010101010101010101010101010E6\n') + hexfile = bincopy.BinFile(word_size_bits=16) + hexfile.add_ihex(records) + align = 6 + size = 12 + chunks = hexfile.segments.chunks(size=size, alignment=align, + padding=b'\xff\xff') + chunks = list(chunks) + assert list(chunks[-1]) == [6, b'\x10\x10\x10\x10\xff\xff\xff\xff\x10\x10\x10' + b'\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10' + b'\x10\x10'] + if __name__ == '__main__': unittest.main()
