From: Stefan Herbrechtsmeier stefan.herbrechtsmeier@weidmueller.com

Skip tests which requires python elftools if the tool is not available.

Signed-off-by: Stefan Herbrechtsmeier stefan.herbrechtsmeier@weidmueller.com Reviewed-by: Simon Glass sjg@chromium.org

---

(no changes since v2)

Changes in v2: - Add commit to skip elf tests

tools/binman/elf_test.py | 14 ++++++++++++++ tools/binman/ftest.py | 18 ++++++++++++++++++ 2 files changed, 32 insertions(+)

diff --git a/tools/binman/elf_test.py b/tools/binman/elf_test.py index 5a51c64cfe..75b867c2be 100644 --- a/tools/binman/elf_test.py +++ b/tools/binman/elf_test.py @@ -122,6 +122,8 @@ class TestElf(unittest.TestCase):

def testOutsideFile(self): """Test a symbol which extends outside the entry area is detected""" + if not elf.ELF_TOOLS: + self.skipTest('Python elftools not available') entry = FakeEntry(10) section = FakeSection() elf_fname = self.ElfTestFile('u_boot_binman_syms') @@ -147,6 +149,8 @@ class TestElf(unittest.TestCase): Only 32 and 64 bits are supported, since we need to store an offset into the image. """ + if not elf.ELF_TOOLS: + self.skipTest('Python elftools not available') entry = FakeEntry(10) section = FakeSection() elf_fname =self.ElfTestFile('u_boot_binman_syms_size') @@ -161,6 +165,8 @@ class TestElf(unittest.TestCase): This should produce -1 values for all thress symbols, taking up the first 16 bytes of the image. """ + if not elf.ELF_TOOLS: + self.skipTest('Python elftools not available') entry = FakeEntry(28) section = FakeSection(sym_value=None) elf_fname = self.ElfTestFile('u_boot_binman_syms') @@ -172,6 +178,8 @@ class TestElf(unittest.TestCase):

def testDebug(self): """Check that enabling debug in the elf module produced debug output""" + if not elf.ELF_TOOLS: + self.skipTest('Python elftools not available') try: tout.init(tout.DEBUG) entry = FakeEntry(24) @@ -281,6 +289,8 @@ class TestElf(unittest.TestCase):

def test_read_segments_bad_data(self): """Test for read_loadable_segments() with an invalid ELF file""" + if not elf.ELF_TOOLS: + self.skipTest('Python elftools not available') fname = self.ElfTestFile('embed_data') with self.assertRaises(ValueError) as e: elf.read_loadable_segments(tools.get_bytes(100, 100)) @@ -288,6 +298,8 @@ class TestElf(unittest.TestCase):

def test_get_file_offset(self): """Test GetFileOffset() gives the correct file offset for a symbol""" + if not elf.ELF_TOOLS: + self.skipTest('Python elftools not available') fname = self.ElfTestFile('embed_data') syms = elf.GetSymbols(fname, ['embed']) addr = syms['embed'].address @@ -314,6 +326,8 @@ class TestElf(unittest.TestCase):

def test_get_symbol_from_address(self): """Test GetSymbolFromAddress()""" + if not elf.ELF_TOOLS: + self.skipTest('Python elftools not available') fname = self.ElfTestFile('elf_sections') sym_name = 'calculate' syms = elf.GetSymbols(fname, [sym_name]) diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index fa1f421c05..da9aa9e679 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -4807,6 +4807,8 @@ class TestFunctional(unittest.TestCase):

def testUpdateFdtInElf(self): """Test that we can update the devicetree in an ELF file""" + if not elf.ELF_TOOLS: + self.skipTest('Python elftools not available') infile = elf_fname = self.ElfTestFile('u_boot_binman_embed') outfile = os.path.join(self._indir, 'u-boot.out') begin_sym = 'dtb_embed_begin' @@ -4858,6 +4860,8 @@ class TestFunctional(unittest.TestCase):

def testUpdateFdtInElfNoSyms(self): """Test that missing symbols are detected with --update-fdt-in-elf""" + if not elf.ELF_TOOLS: + self.skipTest('Python elftools not available') infile = elf_fname = self.ElfTestFile('u_boot_binman_embed') outfile = '' begin_sym = 'wrong_begin' @@ -4871,6 +4875,8 @@ class TestFunctional(unittest.TestCase):

def testUpdateFdtInElfTooSmall(self): """Test that an over-large dtb is detected with --update-fdt-in-elf""" + if not elf.ELF_TOOLS: + self.skipTest('Python elftools not available') infile = elf_fname = self.ElfTestFile('u_boot_binman_embed_sm') outfile = os.path.join(self._indir, 'u-boot.out') begin_sym = 'dtb_embed_begin' @@ -5344,6 +5350,8 @@ fdt fdtmap Extract the devicetree blob from the fdtmap

def testFitSplitElf(self): """Test an image with an FIT with an split-elf operation""" + if not elf.ELF_TOOLS: + self.skipTest('Python elftools not available') entry_args = { 'of-list': 'test-fdt1 test-fdt2', 'default-dt': 'test-fdt2', @@ -5421,6 +5429,8 @@ fdt fdtmap Extract the devicetree blob from the fdtmap

def testFitSplitElfBadElf(self): """Test a FIT split-elf operation with an invalid ELF file""" + if not elf.ELF_TOOLS: + self.skipTest('Python elftools not available') TestFunctional._MakeInputFile('bad.elf', tools.get_bytes(100, 100)) entry_args = { 'of-list': 'test-fdt1 test-fdt2', @@ -5440,6 +5450,8 @@ fdt fdtmap Extract the devicetree blob from the fdtmap

def testFitSplitElfBadDirective(self): """Test a FIT split-elf invalid fit,xxx directive in an image node""" + if not elf.ELF_TOOLS: + self.skipTest('Python elftools not available') err = self._check_bad_fit('227_fit_bad_dir.dts') self.assertIn( "Node '/binman/fit': subnode 'images/@atf-SEQ': Unknown directive 'fit,something'", @@ -5447,6 +5459,8 @@ fdt fdtmap Extract the devicetree blob from the fdtmap

def testFitSplitElfBadDirectiveConfig(self): """Test a FIT split-elf with invalid fit,xxx directive in config""" + if not elf.ELF_TOOLS: + self.skipTest('Python elftools not available') err = self._check_bad_fit('228_fit_bad_dir_config.dts') self.assertEqual( "Node '/binman/fit': subnode 'configurations/@config-SEQ': Unknown directive 'fit,config'", @@ -5470,6 +5484,8 @@ fdt fdtmap Extract the devicetree blob from the fdtmap

def testFitSplitElfMissing(self): """Test an split-elf FIT with a missing ELF file""" + if not elf.ELF_TOOLS: + self.skipTest('Python elftools not available') err = self.checkFitSplitElf(allow_missing=True) self.assertRegex( err, @@ -5477,6 +5493,8 @@ fdt fdtmap Extract the devicetree blob from the fdtmap

def testFitSplitElfFaked(self): """Test an split-elf FIT with faked ELF file""" + if not elf.ELF_TOOLS: + self.skipTest('Python elftools not available') err = self.checkFitSplitElf(allow_missing=True, allow_fake_blobs=True) self.assertRegex( err,

From: Stefan Herbrechtsmeier stefan.herbrechtsmeier@weidmueller.com

Forward AddBintools calls to sub entries in cbfs_util to collect bintools of sub entries.

Signed-off-by: Stefan Herbrechtsmeier stefan.herbrechtsmeier@weidmueller.com

---

Changes in v5: - Add commit to forward AddBintools calls

tools/binman/etype/cbfs.py | 5 +++++ 1 file changed, 5 insertions(+)

diff --git a/tools/binman/etype/cbfs.py b/tools/binman/etype/cbfs.py index 4a1837f26c..832f8d038f 100644 --- a/tools/binman/etype/cbfs.py +++ b/tools/binman/etype/cbfs.py @@ -296,3 +296,8 @@ class Entry_cbfs(Entry): # so that child.data is used to pack into the FIP. self.ObtainContents(skip_entry=child) return True + + def AddBintools(self, btools): + super().AddBintools(btools) + for entry in self._entries.values(): + entry.AddBintools(btools)

From: Stefan Herbrechtsmeier stefan.herbrechtsmeier@weidmueller.com

Collect and thereby initialize bintools before any usage but after generation of entries. This is needed to handle bintools for compress and decompress like other bintools.

Signed-off-by: Stefan Herbrechtsmeier stefan.herbrechtsmeier@weidmueller.com

---

Changes in v5: - Add commit to collect bintools before usage

tools/binman/control.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tools/binman/control.py b/tools/binman/control.py index ce57dc7efc..f8b1733b14 100644 --- a/tools/binman/control.py +++ b/tools/binman/control.py @@ -215,6 +215,7 @@ def ReadEntry(image_fname, entry_path, decomp=True): from binman.image import Image

image = Image.FromFile(image_fname) + image.CollectBintools() entry = image.FindEntryPath(entry_path) return entry.ReadData(decomp)

@@ -251,6 +252,7 @@ def ExtractEntries(image_fname, output_fname, outdir, entry_paths, List of EntryInfo records that were written """ image = Image.FromFile(image_fname) + image.CollectBintools()

if alt_format == 'list': ShowAltFormats(image) @@ -370,6 +372,7 @@ def WriteEntry(image_fname, entry_path, data, do_compress=True, """ tout.info("Write entry '%s', file '%s'" % (entry_path, image_fname)) image = Image.FromFile(image_fname) + image.CollectBintools() entry = image.FindEntryPath(entry_path) WriteEntryToImage(image, entry, data, do_compress=do_compress, allow_resize=allow_resize, write_map=write_map) @@ -507,8 +510,8 @@ def PrepareImagesAndDtbs(dtb_fname, select_images, update_fdt, use_expanded): # without changing the device-tree size, thus ensuring that our # entry offsets remain the same. for image in images.values(): - image.CollectBintools() image.gen_entries() + image.CollectBintools() if update_fdt: image.AddMissingProperties(True) image.ProcessFdt(dtb)

From: Stefan Herbrechtsmeier stefan.herbrechtsmeier@weidmueller.com

Add a DecompressData function to entry class to allow override in child classes and to centralize the compress and decompress in a single class.

Signed-off-by: Stefan Herbrechtsmeier stefan.herbrechtsmeier@weidmueller.com

---

Changes in v5: - Add commit to add DecompressData function to entry class

tools/binman/entry.py | 15 +++++++++++++++ tools/binman/etype/section.py | 3 +-- 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/tools/binman/entry.py b/tools/binman/entry.py index 8f846e55b6..6d58410e91 100644 --- a/tools/binman/entry.py +++ b/tools/binman/entry.py @@ -1086,6 +1086,21 @@ features to produce new behaviours. data = comp_util.compress(indata, self.compress) return data

+ def DecompressData(self, indata): + """Decompress data according to the entry's compression method + + Args: + indata: Data to decompress + + Returns: + Decompressed data + """ + data = comp_util.decompress(indata, self.compress) + if self.compress != 'none': + self.uncomp_size = len(data) + self.uncomp_data = data + return data + @classmethod def UseExpanded(cls, node, etype, new_etype): """Check whether to use an expanded entry type diff --git a/tools/binman/etype/section.py b/tools/binman/etype/section.py index 48165abf6a..b9b2f25345 100644 --- a/tools/binman/etype/section.py +++ b/tools/binman/etype/section.py @@ -13,7 +13,6 @@ import concurrent.futures import re import sys

-from binman import comp_util from binman.entry import Entry from binman import state from dtoc import fdt_util @@ -777,7 +776,7 @@ class Entry_section(Entry): data = parent_data[offset:offset + child.size] if decomp: indata = data - data = comp_util.decompress(indata, child.compress) + data = child.DecompressData(indata) if child.uncomp_size: tout.info("%s: Decompressing data size %#x with algo '%s' to data size %#x" % (child.GetPath(), len(indata), child.compress,

From: Stefan Herbrechtsmeier stefan.herbrechtsmeier@weidmueller.com

Disable the compressed data header of the utilities to compress and decompress data. The header is uncommon, not supported by U-Boot and incompatible with external compressed artifacts.

The header was introduced as part of commit eb0f4a4cb402 ("binman: Support replacing data in a cbfs") to allow device tree entries to be larger than the compressed contents.

Signed-off-by: Stefan Herbrechtsmeier stefan.herbrechtsmeier@weidmueller.com

---

Changes in v5: - Disable header in testInvalidCompress function

Changes in v3: - Add commit to disable compressed data header

tools/binman/entry.py | 4 ++-- tools/binman/ftest.py | 12 +++++++----- 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/tools/binman/entry.py b/tools/binman/entry.py index 6d58410e91..ecce721ac6 100644 --- a/tools/binman/entry.py +++ b/tools/binman/entry.py @@ -1083,7 +1083,7 @@ features to produce new behaviours. self.uncomp_data = indata if self.compress != 'none': self.uncomp_size = len(indata) - data = comp_util.compress(indata, self.compress) + data = comp_util.compress(indata, self.compress, with_header=False) return data

def DecompressData(self, indata): @@ -1095,7 +1095,7 @@ features to produce new behaviours. Returns: Decompressed data """ - data = comp_util.decompress(indata, self.compress) + data = comp_util.decompress(indata, self.compress, with_header=False) if self.compress != 'none': self.uncomp_size = len(data) self.uncomp_data = data diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 6e63b7c27a..edee1f3984 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -1967,7 +1967,7 @@ class TestFunctional(unittest.TestCase): self._ResetDtbs()

def _decompress(self, data): - return comp_util.decompress(data, 'lz4') + return comp_util.decompress(data, 'lz4', with_header=False)

def testCompress(self): """Test compression of blobs""" @@ -4427,14 +4427,16 @@ class TestFunctional(unittest.TestCase): rest = base[len(U_BOOT_DATA):]

# Check compressed data - expect1 = comp_util.compress(COMPRESS_DATA + U_BOOT_DATA, 'lz4') + expect1 = comp_util.compress(COMPRESS_DATA + U_BOOT_DATA, 'lz4', + with_header=False) data1 = rest[:len(expect1)] section1 = self._decompress(data1) self.assertEquals(expect1, data1) self.assertEquals(COMPRESS_DATA + U_BOOT_DATA, section1) rest1 = rest[len(expect1):]

- expect2 = comp_util.compress(COMPRESS_DATA + COMPRESS_DATA, 'lz4') + expect2 = comp_util.compress(COMPRESS_DATA + COMPRESS_DATA, 'lz4', + with_header=False) data2 = rest1[:len(expect2)] section2 = self._decompress(data2) self.assertEquals(expect2, data2) @@ -5219,11 +5221,11 @@ fdt fdtmap Extract the devicetree blob from the fdtmap

def testInvalidCompress(self): with self.assertRaises(ValueError) as e: - comp_util.compress(b'', 'invalid') + comp_util.compress(b'', 'invalid', with_header=False) self.assertIn("Unknown algorithm 'invalid'", str(e.exception))

with self.assertRaises(ValueError) as e: - comp_util.decompress(b'1234', 'invalid') + comp_util.decompress(b'1234', 'invalid', with_header=False) self.assertIn("Unknown algorithm 'invalid'", str(e.exception))

def testBintoolDocs(self):

From: Stefan Herbrechtsmeier stefan.herbrechtsmeier@weidmueller.com

Move compression bintools (packer) creation into test setup to reuse bintool objects between tests.

Signed-off-by: Stefan Herbrechtsmeier stefan.herbrechtsmeier@weidmueller.com

---

Changes in v5: - Add commit to move compression bintools creation into test setup

tools/binman/ftest.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 1538ca3c4e..7ab9289a88 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -23,7 +23,6 @@ import urllib.error from binman import bintool from binman import cbfs_util from binman import cmdline -from binman import comp_util from binman import control from binman import elf from binman import elf_test @@ -107,6 +106,8 @@ BASE_DTB_PROPS = ['offset', 'size', 'image-pos'] # Extra properties expected to be in the device tree when allow-repack is used REPACK_DTB_PROPS = ['orig-offset', 'orig-size']

+# Supported compression bintools +COMP_BINTOOLS = ['lz4', 'lzma_alone']

class TestFunctional(unittest.TestCase): """Functional tests for binman @@ -212,7 +213,9 @@ class TestFunctional(unittest.TestCase): TestFunctional._MakeInputFile('tee.elf', tools.read_file(cls.ElfTestFile('elf_sections')))

- cls.have_lz4 = comp_util.HAVE_LZ4 + cls.comp_bintools = {} + for name in COMP_BINTOOLS: + cls.comp_bintools[name] = bintool.Bintool.create(name)

@classmethod def tearDownClass(cls): @@ -242,9 +245,13 @@ class TestFunctional(unittest.TestCase): cls.toolpath = toolpath cls.verbosity = verbosity

+ def _CheckBintool(self, bintool): + if not bintool.is_present(): + self.skipTest('%s not available' % bintool.name) + def _CheckLz4(self): - if not self.have_lz4: - self.skipTest('lz4 --no-frame-crc not available') + bintool = self.comp_bintools['lz4'] + self._CheckBintool(bintool)

def _CleanupOutputDir(self): """Remove the temporary output directory""" @@ -1967,7 +1974,8 @@ class TestFunctional(unittest.TestCase): self._ResetDtbs()

def _decompress(self, data): - return comp_util.decompress(data, 'lz4') + bintool = self.comp_bintools['lz4'] + return bintool.decompress(data)

def testCompress(self): """Test compression of blobs""" @@ -2855,8 +2863,10 @@ class TestFunctional(unittest.TestCase):

def testExtractCbfsRaw(self): """Test extracting CBFS compressed data without decompressing it""" + bintool = self.comp_bintools['lzma_alone'] + self._CheckBintool(bintool) data = self._RunExtractCmd('section/cbfs/u-boot-dtb', decomp=False) - dtb = comp_util.decompress(data, 'lzma') + dtb = bintool.decompress(data) self.assertEqual(EXTRACT_DTB_SIZE, len(dtb))

def testExtractBadEntry(self): @@ -4427,14 +4437,15 @@ class TestFunctional(unittest.TestCase): rest = base[len(U_BOOT_DATA):]

# Check compressed data - expect1 = comp_util.compress(COMPRESS_DATA + U_BOOT_DATA, 'lz4') + bintool = self.comp_bintools['lz4'] + expect1 = bintool.compress(COMPRESS_DATA + U_BOOT_DATA) data1 = rest[:len(expect1)] section1 = self._decompress(data1) self.assertEquals(expect1, data1) self.assertEquals(COMPRESS_DATA + U_BOOT_DATA, section1) rest1 = rest[len(expect1):]

- expect2 = comp_util.compress(COMPRESS_DATA + COMPRESS_DATA, 'lz4') + expect2 = bintool.compress(COMPRESS_DATA + COMPRESS_DATA) data2 = rest1[:len(expect2)] section2 = self._decompress(data2) self.assertEquals(expect2, data2)

From: Stefan Herbrechtsmeier stefan.herbrechtsmeier@weidmueller.com

Move management of the bintool to compress and decompress data into the entry class and add the bintool to the list of required bintools.

Signed-off-by: Stefan Herbrechtsmeier stefan.herbrechtsmeier@weidmueller.com

---

Changes in v5: - Add commit to move compression bintool management into entry class

tools/binman/comp_util.py | 69 ------------------- tools/binman/entry.py | 23 +++++-- tools/binman/ftest.py | 15 ++-- .../binman/test/237_compress_dtb_invalid.dts | 16 +++++ 4 files changed, 41 insertions(+), 82 deletions(-) delete mode 100644 tools/binman/comp_util.py create mode 100644 tools/binman/test/237_compress_dtb_invalid.dts

diff --git a/tools/binman/comp_util.py b/tools/binman/comp_util.py deleted file mode 100644 index 269bbf7975..0000000000 --- a/tools/binman/comp_util.py +++ /dev/null @@ -1,69 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0+ -# Copyright 2022 Google LLC -# -"""Utilities to compress and decompress data""" - -import tempfile - -from binman import bintool -from patman import tools - -LZ4 = bintool.Bintool.create('lz4') -HAVE_LZ4 = LZ4.is_present() - -LZMA_ALONE = bintool.Bintool.create('lzma_alone') -HAVE_LZMA_ALONE = LZMA_ALONE.is_present() - - -def compress(indata, algo): - """Compress some data using a given algorithm - - Note that for lzma this uses an old version of the algorithm, not that - provided by xz. - - This requires 'lz4' and 'lzma_alone' tools. It also requires an output - directory to be previously set up, by calling PrepareOutputDir(). - - Args: - indata (bytes): Input data to compress - algo (str): Algorithm to use ('none', 'lz4' or 'lzma') - - Returns: - bytes: Compressed data - """ - if algo == 'none': - return indata - if algo == 'lz4': - data = LZ4.compress(indata) - # cbfstool uses a very old version of lzma - elif algo == 'lzma': - data = LZMA_ALONE.compress(indata) - else: - raise ValueError("Unknown algorithm '%s'" % algo) - return data - -def decompress(indata, algo): - """Decompress some data using a given algorithm - - Note that for lzma this uses an old version of the algorithm, not that - provided by xz. - - This requires 'lz4' and 'lzma_alone' tools. It also requires an output - directory to be previously set up, by calling PrepareOutputDir(). - - Args: - indata (bytes): Input data to decompress - algo (str): Algorithm to use ('none', 'lz4' or 'lzma') - - Returns: - (bytes) Compressed data - """ - if algo == 'none': - return indata - if algo == 'lz4': - data = LZ4.decompress(indata) - elif algo == 'lzma': - data = LZMA_ALONE.decompress(indata) - else: - raise ValueError("Unknown algorithm '%s'" % algo) - return data diff --git a/tools/binman/entry.py b/tools/binman/entry.py index 398d5caa39..6a6c3e6bb4 100644 --- a/tools/binman/entry.py +++ b/tools/binman/entry.py @@ -11,7 +11,6 @@ import pathlib import sys

from binman import bintool -from binman import comp_util from dtoc import fdt_util from patman import tools from patman.tools import to_hex, to_hex_size @@ -82,6 +81,7 @@ class Entry(object): missing_bintools: List of missing bintools for this entry update_hash: True if this entry's "hash" subnode should be updated with a hash of the entry contents + comp_bintool: Bintools used for compress and decompress data """ def __init__(self, section, etype, node, name_prefix=''): # Put this here to allow entry-docs and help to work without libfdt @@ -116,6 +116,7 @@ class Entry(object): self.bintools = {} self.missing_bintools = [] self.update_hash = True + self.comp_bintool = None

@staticmethod def FindEntryClass(etype, expanded): @@ -1083,7 +1084,9 @@ features to produce new behaviours. self.uncomp_data = indata if self.compress != 'none': self.uncomp_size = len(indata) - data = comp_util.compress(indata, self.compress) + data = self.comp_bintool.compress(indata) + else: + data = indata return data

def DecompressData(self, indata): @@ -1095,9 +1098,11 @@ features to produce new behaviours. Returns: Decompressed data """ - data = comp_util.decompress(indata, self.compress) if self.compress != 'none': self.uncomp_size = len(data) + data = self.comp_bintool.decompress(indata) + else: + data = indata self.uncomp_data = data return data

@@ -1138,8 +1143,18 @@ features to produce new behaviours.

Args: btools (dict of Bintool): + + Raise: + ValueError if compression algorithm is not supported """ - pass + algo = self.compress + if algo != 'none': + algos = ['lz4', 'lzma'] + if algo not in algos: + raise ValueError("Unknown algorithm '%s'" % algo) + names = {'lzma': 'lzma_alone'} + name = names.get(self.compress, self.compress) + self.comp_bintool = self.AddBintool(btools, name)

@classmethod def AddBintool(self, tools, name): diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 7ab9289a88..939b265d7c 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -5228,15 +5228,6 @@ fdt fdtmap Extract the devicetree blob from the fdtmap self._DoBinman(*args) self.assertIn('failed to fetch with all methods', stdout.getvalue())

- def testInvalidCompress(self): - with self.assertRaises(ValueError) as e: - comp_util.compress(b'', 'invalid') - self.assertIn("Unknown algorithm 'invalid'", str(e.exception)) - - with self.assertRaises(ValueError) as e: - comp_util.decompress(b'1234', 'invalid') - self.assertIn("Unknown algorithm 'invalid'", str(e.exception)) - def testBintoolDocs(self): """Test for creation of bintool documentation""" with test_util.capture_sys_output() as (stdout, stderr): @@ -5773,6 +5764,12 @@ fdt fdtmap Extract the devicetree blob from the fdtmap orig2 = self._decompress(comp_data) self.assertEqual(orig, orig2)

+ def testInvalidCompress(self): + """Test that invalid compress algorithm is detected""" + with self.assertRaises(ValueError) as e: + self._DoTestFile('237_compress_dtb_invalid.dts') + self.assertIn("Unknown algorithm 'invalid'", str(e.exception)) +

if __name__ == "__main__": unittest.main() diff --git a/tools/binman/test/237_compress_dtb_invalid.dts b/tools/binman/test/237_compress_dtb_invalid.dts new file mode 100644 index 0000000000..228139060b --- /dev/null +++ b/tools/binman/test/237_compress_dtb_invalid.dts @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + u-boot { + }; + u-boot-dtb { + compress = "invalid"; + }; + }; +};

From: Stefan Herbrechtsmeier stefan.herbrechtsmeier@weidmueller.com

Add common test functions to test all supported compressions.

Signed-off-by: Stefan Herbrechtsmeier stefan.herbrechtsmeier@weidmueller.com Reviewed-by: Simon Glass sjg@chromium.org

--- Instead of the for loop it is possible to use Parameterized [1] testing.

[1] https://github.com/wolever/parameterized

Changes in v5: - Use self.comp_bintools dict - Use _CheckBintool(bintool) function

Changes in v3: - Use 'tools.get_bytes(0, 64)' instead of 'bytes([0]) * 64' - Check if tool is present - Rename tests

Changes in v2: - Add commit to add compression tests

tools/binman/ftest.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+)

diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index ffcd7d2567..bbb2fa19c1 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -5779,6 +5779,32 @@ fdt fdtmap Extract the devicetree blob from the fdtmap self._DoTestFile('237_compress_dtb_invalid.dts') self.assertIn("Unknown algorithm 'invalid'", str(e.exception))

+ def testCompUtilCompressions(self): + """Test compression algorithms""" + for bintool in self.comp_bintools.values(): + self._CheckBintool(bintool) + data = bintool.compress(COMPRESS_DATA) + self.assertNotEqual(COMPRESS_DATA, data) + orig = bintool.decompress(data) + self.assertEquals(COMPRESS_DATA, orig) + + def testCompUtilVersions(self): + """Test tool version of compression algorithms""" + for bintool in self.comp_bintools.values(): + self._CheckBintool(bintool) + version = bintool.version() + self.assertRegex(version, '^v?[0-9]+[0-9.]*') + + def testCompUtilPadding(self): + """Test padding of compression algorithms""" + for bintool in self.comp_bintools.values(): + self._CheckBintool(bintool) + data = bintool.compress(COMPRESS_DATA) + self.assertNotEqual(COMPRESS_DATA, data) + data += tools.get_bytes(0, 64) + orig = bintool.decompress(data) + self.assertEquals(COMPRESS_DATA, orig) +

if __name__ == "__main__": unittest.main()

From: Stefan Herbrechtsmeier stefan.herbrechtsmeier@weidmueller.com

Add bzip2 bintool to binman to support on-the-fly compression.

Signed-off-by: Stefan Herbrechtsmeier stefan.herbrechtsmeier@weidmueller.com

---

Changes in v5: - Add algorithm name to entry.py - Add tool name to ftest.py

Changes in v2: - Add commit to add bzip2 bintool

tools/binman/btool/bzip2.py | 30 ++++++++++++++++++++++++++++++ tools/binman/entry.py | 2 +- tools/binman/ftest.py | 2 +- 3 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 tools/binman/btool/bzip2.py

diff --git a/tools/binman/btool/bzip2.py b/tools/binman/btool/bzip2.py new file mode 100644 index 0000000000..9be87a621f --- /dev/null +++ b/tools/binman/btool/bzip2.py @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (C) 2022 Weidmüller Interface GmbH & Co. KG +# Stefan Herbrechtsmeier stefan.herbrechtsmeier@weidmueller.com +# +"""Bintool implementation for bzip2 + +bzip2 allows compression and decompression of files. + +Documentation is available via:: + + man bzip2 +""" + +from binman import bintool + +# pylint: disable=C0103 +class Bintoolbzip2(bintool.BintoolPacker): + """Compression/decompression using the bzip2 algorithm + + This bintool supports running `bzip2` to compress and decompress data, as + used by binman. + + It is also possible to fetch the tool, which uses `apt` to install it. + + Documentation is available via:: + + man bzip2 + """ + def __init__(self, name): + super().__init__(name, version_regex=r'bzip2.*Version ([0-9.]+)') diff --git a/tools/binman/entry.py b/tools/binman/entry.py index 31389be672..5fb8517329 100644 --- a/tools/binman/entry.py +++ b/tools/binman/entry.py @@ -1157,7 +1157,7 @@ features to produce new behaviours. """ algo = self.compress if algo != 'none': - algos = ['lz4', 'lzma'] + algos = ['bzip2', 'lz4', 'lzma'] if algo not in algos: raise ValueError("Unknown algorithm '%s'" % algo) names = {'lzma': 'lzma_alone'} diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index bbb2fa19c1..9fa8e9e049 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -107,7 +107,7 @@ BASE_DTB_PROPS = ['offset', 'size', 'image-pos'] REPACK_DTB_PROPS = ['orig-offset', 'orig-size']

# Supported compression bintools -COMP_BINTOOLS = ['lz4', 'lzma_alone'] +COMP_BINTOOLS = ['bzip2', 'lz4', 'lzma_alone']

class TestFunctional(unittest.TestCase): """Functional tests for binman

From: Stefan Herbrechtsmeier stefan.herbrechtsmeier@weidmueller.com

Add lzop bintool to binman to support on-the-fly compression.

Signed-off-by: Stefan Herbrechtsmeier stefan.herbrechtsmeier@weidmueller.com

---

Changes in v5: - Add algorithm name to entry.py - Add tool name to entry.py and ftest.py

Changes in v2: - Add commit to add lzop bintool

tools/binman/btool/lzop.py | 30 ++++++++++++++++++++++++++++++ tools/binman/entry.py | 4 ++-- tools/binman/ftest.py | 2 +- 3 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 tools/binman/btool/lzop.py

diff --git a/tools/binman/btool/lzop.py b/tools/binman/btool/lzop.py new file mode 100644 index 0000000000..f6903b4db7 --- /dev/null +++ b/tools/binman/btool/lzop.py @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (C) 2022 Weidmüller Interface GmbH & Co. KG +# Stefan Herbrechtsmeier stefan.herbrechtsmeier@weidmueller.com +# +"""Bintool implementation for lzop + +lzop allows compression and decompression of files. + +Documentation is available via:: + + man lzop +""" + +from binman import bintool + +# pylint: disable=C0103 +class Bintoollzop(bintool.BintoolPacker): + """Compression/decompression using the lzop algorithm + + This bintool supports running `lzop` to compress and decompress data, as + used by binman. + + It is also possible to fetch the tool, which uses `apt` to install it. + + Documentation is available via:: + + man lzop + """ + def __init__(self, name): + super().__init__(name, 'lzo', compress_args=[]) diff --git a/tools/binman/entry.py b/tools/binman/entry.py index 8562a65d30..ae3e94a167 100644 --- a/tools/binman/entry.py +++ b/tools/binman/entry.py @@ -1157,10 +1157,10 @@ features to produce new behaviours. """ algo = self.compress if algo != 'none': - algos = ['bzip2', 'gzip', 'lz4', 'lzma'] + algos = ['bzip2', 'gzip', 'lz4', 'lzma', 'lzo'] if algo not in algos: raise ValueError("Unknown algorithm '%s'" % algo) - names = {'lzma': 'lzma_alone'} + names = {'lzma': 'lzma_alone', 'lzo': 'lzop'} name = names.get(self.compress, self.compress) self.comp_bintool = self.AddBintool(btools, name)

diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 911c955cba..2cac505d8b 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -107,7 +107,7 @@ BASE_DTB_PROPS = ['offset', 'size', 'image-pos'] REPACK_DTB_PROPS = ['orig-offset', 'orig-size']

# Supported compression bintools -COMP_BINTOOLS = ['bzip2', 'gzip', 'lz4', 'lzma_alone'] +COMP_BINTOOLS = ['bzip2', 'gzip', 'lz4', 'lzma_alone', 'lzop']

class TestFunctional(unittest.TestCase): """Functional tests for binman

From: Stefan Herbrechtsmeier stefan.herbrechtsmeier@weidmueller.com

Add zstd bintool to binman to support on-the-fly compression.

Signed-off-by: Stefan Herbrechtsmeier stefan.herbrechtsmeier@weidmueller.com

---

Changes in v5: - Test that zstd compress of device-tree files failed - Add algorithm name to entry.py - Add tool name to ftest.py

Changes in v2: - Add commit to add zstd bintool

tools/binman/btool/zstd.py | 30 +++++++++++++++++++++ tools/binman/entry.py | 2 +- tools/binman/etype/blob_dtb.py | 4 +++ tools/binman/ftest.py | 12 +++++++-- tools/binman/test/238_compress_dtb_zstd.dts | 16 +++++++++++ 5 files changed, 61 insertions(+), 3 deletions(-) create mode 100644 tools/binman/btool/zstd.py create mode 100644 tools/binman/test/238_compress_dtb_zstd.dts

diff --git a/tools/binman/btool/zstd.py b/tools/binman/btool/zstd.py new file mode 100644 index 0000000000..299bd37126 --- /dev/null +++ b/tools/binman/btool/zstd.py @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (C) 2022 Weidmüller Interface GmbH & Co. KG +# Stefan Herbrechtsmeier stefan.herbrechtsmeier@weidmueller.com +# +"""Bintool implementation for zstd + +zstd allows compression and decompression of files. + +Documentation is available via:: + + man zstd +""" + +from binman import bintool + +# pylint: disable=C0103 +class Bintoolzstd(bintool.BintoolPacker): + """Compression/decompression using the zstd algorithm + + This bintool supports running `zstd` to compress and decompress data, as + used by binman. + + It is also possible to fetch the tool, which uses `apt` to install it. + + Documentation is available via:: + + man zstd + """ + def __init__(self, name): + super().__init__(name) diff --git a/tools/binman/entry.py b/tools/binman/entry.py index 0d4990cf5d..8e4a5e301a 100644 --- a/tools/binman/entry.py +++ b/tools/binman/entry.py @@ -1157,7 +1157,7 @@ features to produce new behaviours. """ algo = self.compress if algo != 'none': - algos = ['bzip2', 'gzip', 'lz4', 'lzma', 'lzo', 'xz'] + algos = ['bzip2', 'gzip', 'lz4', 'lzma', 'lzo', 'xz', 'zstd'] if algo not in algos: raise ValueError("Unknown algorithm '%s'" % algo) names = {'lzma': 'lzma_alone', 'lzo': 'lzop'} diff --git a/tools/binman/etype/blob_dtb.py b/tools/binman/etype/blob_dtb.py index 5a6a454748..6a3fbc4775 100644 --- a/tools/binman/etype/blob_dtb.py +++ b/tools/binman/etype/blob_dtb.py @@ -47,6 +47,10 @@ class Entry_blob_dtb(Entry_blob): def ProcessContents(self): """Re-read the DTB contents so that we get any calculated properties""" _, indata = state.GetFdtContents(self.GetFdtEtype()) + + if self.compress == 'zstd' and self.prepend != 'length': + self.Raise('The zstd compression requires a length header') + data = self.CompressData(indata) return self.ProcessContentsUpdate(data)

diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index b337e2168c..134301365e 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -107,7 +107,7 @@ BASE_DTB_PROPS = ['offset', 'size', 'image-pos'] REPACK_DTB_PROPS = ['orig-offset', 'orig-size']

# Supported compression bintools -COMP_BINTOOLS = ['bzip2', 'gzip', 'lz4', 'lzma_alone', 'lzop', 'xz'] +COMP_BINTOOLS = ['bzip2', 'gzip', 'lz4', 'lzma_alone', 'lzop', 'xz', 'zstd']

class TestFunctional(unittest.TestCase): """Functional tests for binman @@ -5797,7 +5797,8 @@ fdt fdtmap Extract the devicetree blob from the fdtmap

def testCompUtilPadding(self): """Test padding of compression algorithms""" - for bintool in self.comp_bintools.values(): + # Skip zstd because it doesn't support padding + for bintool in [v for k,v in self.comp_bintools.items() if k != 'zstd']: self._CheckBintool(bintool) data = bintool.compress(COMPRESS_DATA) self.assertNotEqual(COMPRESS_DATA, data) @@ -5805,6 +5806,13 @@ fdt fdtmap Extract the devicetree blob from the fdtmap orig = bintool.decompress(data) self.assertEquals(COMPRESS_DATA, orig)

+ def testCompressDtbZstd(self): + """Test that zstd compress of device-tree files failed""" + with self.assertRaises(ValueError) as e: + self._DoTestFile('238_compress_dtb_zstd.dts') + self.assertIn("Node '/binman/u-boot-dtb': The zstd compression " + "requires a length header", str(e.exception)) +

if __name__ == "__main__": unittest.main() diff --git a/tools/binman/test/238_compress_dtb_zstd.dts b/tools/binman/test/238_compress_dtb_zstd.dts new file mode 100644 index 0000000000..90cf85d1e2 --- /dev/null +++ b/tools/binman/test/238_compress_dtb_zstd.dts @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + u-boot { + }; + u-boot-dtb { + compress = "zstd"; + }; + }; +};