Add support for compressing blob entries. This can help reduce image sizes for many types of data. It requires that the firmware be able to decompress the data at run-time.
Signed-off-by: Simon Glass sjg@chromium.org ---
tools/binman/README | 16 ++++++++++ tools/binman/README.entries | 7 +++++ tools/binman/etype/blob.py | 49 +++++++++++++++++++++++++------ tools/binman/ftest.py | 32 ++++++++++++++++++++ tools/binman/test/83_compress.dts | 11 +++++++ 5 files changed, 106 insertions(+), 9 deletions(-) create mode 100644 tools/binman/test/83_compress.dts
diff --git a/tools/binman/README b/tools/binman/README index 3dc8b9b80a8..34b83110f64 100644 --- a/tools/binman/README +++ b/tools/binman/README @@ -593,6 +593,22 @@ the device tree. These can be used by U-Boot at run-time to find the location of each entry.
+Compression +----------- + +Binman support compression for 'blob' entries (those of type 'blob' and +derivatives). To enable this for an entry, add a 'compression' property: + + blob { + filename = "datafile"; + compression = "lz4"; + }; + +The entry will then contain the compressed data, using the 'lz4' compression +algorithm. Currently this is the only one that is supported. + + + Map files ---------
diff --git a/tools/binman/README.entries b/tools/binman/README.entries index 091fb5ce2b6..2cf7dc0338d 100644 --- a/tools/binman/README.entries +++ b/tools/binman/README.entries @@ -19,11 +19,18 @@ class by other entry types.
Properties / Entry arguments: - filename: Filename of file to read into entry + - compress: Compression algorithm to use: + none: No compression + lz4: Use lz4 compression (via 'lz4' command-line utility)
This entry reads data from a file and places it in the entry. The default filename is often specified specified by the subclass. See for example the 'u_boot' entry which provides the filename 'u-boot.bin'.
+If compression is enabled, an extra 'uncomp-size' property is written to +the node (if enabled with -u) which provides the uncompressed size of the +data. +
Entry: blob-dtb: A blob that holds a device tree diff --git a/tools/binman/etype/blob.py b/tools/binman/etype/blob.py index 3f46eecf308..642a0e482a7 100644 --- a/tools/binman/etype/blob.py +++ b/tools/binman/etype/blob.py @@ -7,6 +7,7 @@
from entry import Entry import fdt_util +import state import tools
class Entry_blob(Entry): @@ -17,14 +18,23 @@ class Entry_blob(Entry):
Properties / Entry arguments: - filename: Filename of file to read into entry + - compress: Compression algorithm to use: + none: No compression + lz4: Use lz4 compression (via 'lz4' command-line utility)
This entry reads data from a file and places it in the entry. The default filename is often specified specified by the subclass. See for example the 'u_boot' entry which provides the filename 'u-boot.bin'. + + If compression is enabled, an extra 'uncomp-size' property is written to + the node (if enabled with -u) which provides the uncompressed size of the + data. """ def __init__(self, section, etype, node): Entry.__init__(self, section, etype, node) - self._filename = fdt_util.GetString(self._node, "filename", self.etype) + self._filename = fdt_util.GetString(self._node, 'filename', self.etype) + self._compress = fdt_util.GetString(self._node, 'compress', 'none') + self._uncompressed_size = None
def ObtainContents(self): self._filename = self.GetDefaultFilename() @@ -33,15 +43,36 @@ class Entry_blob(Entry): return True
def ReadBlobContents(self): - with open(self._pathname) as fd: - # We assume the data is small enough to fit into memory. If this - # is used for large filesystem image that might not be true. - # In that case, Image.BuildImage() could be adjusted to use a - # new Entry method which can read in chunks. Then we could copy - # the data in chunks and avoid reading it all at once. For now - # this seems like an unnecessary complication. - self.SetContents(fd.read()) + # We assume the data is small enough to fit into memory. If this + # is used for large filesystem image that might not be true. + # In that case, Image.BuildImage() could be adjusted to use a + # new Entry method which can read in chunks. Then we could copy + # the data in chunks and avoid reading it all at once. For now + # this seems like an unnecessary complication. + data = tools.ReadFile(self._pathname) + if self._compress == 'lz4': + self._uncompressed_size = len(data) + ''' + import lz4 # Import this only if needed (python-lz4 dependency) + + try: + data = lz4.frame.compress(data) + except AttributeError: + data = lz4.compress(data) + ''' + data = tools.Run('lz4', '-c', self._pathname, ) + self.SetContents(data) return True
def GetDefaultFilename(self): return self._filename + + def AddMissingProperties(self): + Entry.AddMissingProperties(self) + if self._compress != 'none': + state.AddZeroProp(self._node, 'uncomp-size') + + def SetCalculatedProperties(self): + Entry.SetCalculatedProperties(self) + if self._uncompressed_size is not None: + state.SetInt(self._node, 'uncomp-size', self._uncompressed_size) diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 6bfef7b63a6..c5e7236c424 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -6,6 +6,7 @@ # # python -m unittest func_test.TestFunctional.testHelp
+import lz4 from optparse import OptionParser import os import shutil @@ -54,6 +55,7 @@ CROS_EC_RW_DATA = 'ecrw' GBB_DATA = 'gbbd' BMPBLK_DATA = 'bmp' VBLOCK_DATA = 'vblk' +COMPRESS_DATA = 'data to compress'
class TestFunctional(unittest.TestCase): @@ -116,6 +118,8 @@ class TestFunctional(unittest.TestCase): with open(self.TestFile('descriptor.bin')) as fd: TestFunctional._MakeInputFile('descriptor.bin', fd.read())
+ TestFunctional._MakeInputFile('compress', COMPRESS_DATA) + @classmethod def tearDownClass(self): """Remove the temporary input directory and its contents""" @@ -1505,6 +1509,34 @@ class TestFunctional(unittest.TestCase): finally: self._ResetDtbs()
+ def _decompress(self, data): + out = os.path.join(self._indir, 'lz4.tmp') + with open(out, 'wb') as fd: + fd.write(data) + return tools.Run('lz4', '-dc', out) + ''' + try: + orig = lz4.frame.decompress(data) + except AttributeError: + orig = lz4.decompress(data) + ''' + + def testCompress(self): + """Test compression of blobs""" + data, _, _, out_dtb_fname = self._DoReadFileDtb('83_compress.dts', + use_real_dtb=True, update_dtb=True) + dtb = fdt.Fdt(out_dtb_fname) + dtb.Scan() + props = self._GetPropTree(dtb, ['size', 'uncomp-size']) + orig = self._decompress(data) + self.assertEquals(COMPRESS_DATA, orig) + expected = { + 'blob:uncomp-size': len(COMPRESS_DATA), + 'blob:size': len(data), + 'size': len(data), + } + self.assertEqual(expected, props) +
if __name__ == "__main__": unittest.main() diff --git a/tools/binman/test/83_compress.dts b/tools/binman/test/83_compress.dts new file mode 100644 index 00000000000..07813bdeaa3 --- /dev/null +++ b/tools/binman/test/83_compress.dts @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0+ +/dts-v1/; + +/ { + binman { + blob { + filename = "compress"; + compress = "lz4"; + }; + }; +};