At present the order of struct field emitted by this tool depends on the internal workings of a Python dictionary. Sort the fields to remove this uncertainty, so that tests are deterministic.

Signed-off-by: Simon Glass sjg@chromium.org ---

tools/dtoc/dtb_platdata.py | 3 ++- tools/dtoc/test_dtoc.py | 12 ++++++------ 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/tools/dtoc/dtb_platdata.py b/tools/dtoc/dtb_platdata.py index 4aeeab6fba9..3b66af8df78 100644 --- a/tools/dtoc/dtb_platdata.py +++ b/tools/dtoc/dtb_platdata.py @@ -464,7 +464,8 @@ class DtbPlatdata(object): var_name = conv_name_to_c(node.name) self.buf('static const struct %s%s %s%s = {\n' % (STRUCT_PREFIX, struct_name, VAL_PREFIX, var_name)) - for pname, prop in node.props.items(): + for pname in sorted(node.props): + prop = node.props[pname] if pname in PROP_IGNORE_LIST or pname[0] == '#': continue member_name = conv_name_to_c(prop.name) diff --git a/tools/dtoc/test_dtoc.py b/tools/dtoc/test_dtoc.py index ae59a0a52a1..b915b278560 100644 --- a/tools/dtoc/test_dtoc.py +++ b/tools/dtoc/test_dtoc.py @@ -199,16 +199,16 @@ struct dtd_sandbox_spl_test_2 { data = infile.read() self._CheckStrings(C_HEADER + ''' static const struct dtd_sandbox_spl_test dtv_spl_test = { +\t.boolval\t\t= true, \t.bytearray\t\t= {0x6, 0x0, 0x0}, \t.byteval\t\t= 0x5, +\t.intarray\t\t= {0x2, 0x3, 0x4, 0x0}, \t.intval\t\t\t= 0x1, -\t.notstring\t\t= {0x20, 0x21, 0x22, 0x10, 0x0}, \t.longbytearray\t\t= {0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0x10, \t\t0x11}, -\t.stringval\t\t= "message", -\t.boolval\t\t= true, -\t.intarray\t\t= {0x2, 0x3, 0x4, 0x0}, +\t.notstring\t\t= {0x20, 0x21, 0x22, 0x10, 0x0}, \t.stringarray\t\t= {"multi-word", "message", ""}, +\t.stringval\t\t= "message", }; U_BOOT_DEVICE(spl_test) = { \t.name\t\t= "sandbox_spl_test", @@ -219,12 +219,12 @@ U_BOOT_DEVICE(spl_test) = { static const struct dtd_sandbox_spl_test dtv_spl_test2 = { \t.bytearray\t\t= {0x1, 0x23, 0x34}, \t.byteval\t\t= 0x8, +\t.intarray\t\t= {0x5, 0x0, 0x0, 0x0}, \t.intval\t\t\t= 0x3, \t.longbytearray\t\t= {0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, \t\t0x0}, -\t.stringval\t\t= "message2", -\t.intarray\t\t= {0x5, 0x0, 0x0, 0x0}, \t.stringarray\t\t= {"another", "multi-word", "message"}, +\t.stringval\t\t= "message2", }; U_BOOT_DEVICE(spl_test2) = { \t.name\t\t= "sandbox_spl_test",

This method does not actually use any members of the Prop class. Move it out of the class so that it is easier to add unit tests.

Signed-off-by: Simon Glass sjg@chromium.org ---

tools/dtoc/fdt.py | 104 +++++++++++++++++++++++----------------------- 1 file changed, 53 insertions(+), 51 deletions(-)

diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt.py index 9518a287a26..35453fbed9a 100644 --- a/tools/dtoc/fdt.py +++ b/tools/dtoc/fdt.py @@ -29,6 +29,57 @@ def CheckErr(errnum, msg): raise ValueError('Error %d: %s: %s' % (errnum, libfdt.fdt_strerror(errnum), msg))

+ +def BytesToValue(bytes): + """Converts a string of bytes into a type and value + + Args: + A string containing bytes + + Return: + A tuple: + Type of data + Data, either a single element or a list of elements. Each element + is one of: + TYPE_STRING: string value from the property + TYPE_INT: a byte-swapped integer stored as a 4-byte string + TYPE_BYTE: a byte stored as a single-byte string + """ + bytes = str(bytes) + size = len(bytes) + strings = bytes.split('\0') + is_string = True + count = len(strings) - 1 + if count > 0 and not strings[-1]: + for string in strings[:-1]: + if not string: + is_string = False + break + for ch in string: + if ch < ' ' or ch > '~': + is_string = False + break + else: + is_string = False + if is_string: + if count == 1: + return TYPE_STRING, strings[0] + else: + return TYPE_STRING, strings[:-1] + if size % 4: + if size == 1: + return TYPE_BYTE, bytes[0] + else: + return TYPE_BYTE, list(bytes) + val = [] + for i in range(0, size, 4): + val.append(bytes[i:i + 4]) + if size == 4: + return TYPE_INT, val[0] + else: + return TYPE_INT, val + + class Prop: """A device tree property

@@ -49,7 +100,7 @@ class Prop: self.type = TYPE_BOOL self.value = True return - self.type, self.value = self.BytesToValue(bytes) + self.type, self.value = BytesToValue(bytes)

def RefreshOffset(self, poffset): self._offset = poffset @@ -88,55 +139,6 @@ class Prop: while len(self.value) < len(newprop.value): self.value.append(val)

- def BytesToValue(self, bytes): - """Converts a string of bytes into a type and value - - Args: - A string containing bytes - - Return: - A tuple: - Type of data - Data, either a single element or a list of elements. Each element - is one of: - TYPE_STRING: string value from the property - TYPE_INT: a byte-swapped integer stored as a 4-byte string - TYPE_BYTE: a byte stored as a single-byte string - """ - bytes = str(bytes) - size = len(bytes) - strings = bytes.split('\0') - is_string = True - count = len(strings) - 1 - if count > 0 and not strings[-1]: - for string in strings[:-1]: - if not string: - is_string = False - break - for ch in string: - if ch < ' ' or ch > '~': - is_string = False - break - else: - is_string = False - if is_string: - if count == 1: - return TYPE_STRING, strings[0] - else: - return TYPE_STRING, strings[:-1] - if size % 4: - if size == 1: - return TYPE_BYTE, bytes[0] - else: - return TYPE_BYTE, list(bytes) - val = [] - for i in range(0, size, 4): - val.append(bytes[i:i + 4]) - if size == 4: - return TYPE_INT, val[0] - else: - return TYPE_INT, val - @classmethod def GetEmpty(self, type): """Get an empty / zero value of the given type @@ -183,7 +185,7 @@ class Prop: bytes: New property value to set """ self.bytes = str(bytes) - self.type, self.value = self.BytesToValue(bytes) + self.type, self.value = BytesToValue(bytes) self.dirty = True

def Sync(self, auto_resize=False):

In Python 3 bytes and str are separate types. Use bytes to ensure that the code functions correctly with Python 3.

Signed-off-by: Simon Glass sjg@chromium.org ---

tools/dtoc/fdt.py | 14 +++++++++----- tools/dtoc/test_fdt.py | 18 +++++++++--------- tools/patman/tools.py | 25 +++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 14 deletions(-)

diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt.py index d539248430a..cbd9cbabe31 100644 --- a/tools/dtoc/fdt.py +++ b/tools/dtoc/fdt.py @@ -193,7 +193,7 @@ class Prop: Args: bytes: New property value to set """ - self.bytes = str(bytes) + self.bytes = bytes self.type, self.value = BytesToValue(bytes) self.dirty = True

@@ -398,7 +398,9 @@ class Node: prop_name: Name of property to set val: String value to set (will be \0-terminated in DT) """ - self.props[prop_name].SetData(val + chr(0)) + if sys.version_info[0] >= 3: + val = bytes(val, 'utf-8') + self.props[prop_name].SetData(val + b'\0')

def AddString(self, prop_name, val): """Add a new string property to a node @@ -410,7 +412,9 @@ class Node: prop_name: Name of property to add val: String value of property """ - self.props[prop_name] = Prop(self, None, prop_name, val + chr(0)) + if sys.version_info[0] >= 3: + val = bytes(val, 'utf-8') + self.props[prop_name] = Prop(self, None, prop_name, val + b'\0')

def AddSubnode(self, name): """Add a new subnode to the node @@ -496,7 +500,7 @@ class Fdt: Fdt object containing the data """ fdt = Fdt(None) - fdt._fdt_obj = libfdt.Fdt(bytearray(data)) + fdt._fdt_obj = libfdt.Fdt(bytes(data)) return fdt

def LookupPhandle(self, phandle): @@ -586,7 +590,7 @@ class Fdt: Returns: The FDT contents as a string of bytes """ - return self._fdt_obj.as_bytearray() + return bytes(self._fdt_obj.as_bytearray())

def GetFdtObj(self): """Get the contents of the FDT diff --git a/tools/dtoc/test_fdt.py b/tools/dtoc/test_fdt.py index 79f97d8013c..3cd34b745ed 100755 --- a/tools/dtoc/test_fdt.py +++ b/tools/dtoc/test_fdt.py @@ -47,7 +47,7 @@ def _GetPropertyValue(dtb, node, prop_name): # Add 12, which is sizeof(struct fdt_property), to get to start of data offset = prop.GetOffset() + 12 data = dtb.GetContents()[offset:offset + len(prop.value)] - return prop, [chr(x) for x in data] + return prop, [tools.ToChar(x) for x in data]

class TestFdt(unittest.TestCase): @@ -383,7 +383,7 @@ class TestProp(unittest.TestCase): self.node.AddString('string', val) self.dtb.Sync(auto_resize=True) data = self.fdt.getprop(self.node.Offset(), 'string') - self.assertEqual(val + '\0', data) + self.assertEqual(tools.ToBytes(val) + b'\0', data)

self.fdt.pack() self.node.SetString('string', val + 'x') @@ -393,21 +393,21 @@ class TestProp(unittest.TestCase): self.node.SetString('string', val[:-1])

prop = self.node.props['string'] - prop.SetData(val) + prop.SetData(tools.ToBytes(val)) self.dtb.Sync(auto_resize=False) data = self.fdt.getprop(self.node.Offset(), 'string') - self.assertEqual(val, data) + self.assertEqual(tools.ToBytes(val), data)

self.node.AddEmptyProp('empty', 5) self.dtb.Sync(auto_resize=True) prop = self.node.props['empty'] - prop.SetData(val) + prop.SetData(tools.ToBytes(val)) self.dtb.Sync(auto_resize=False) data = self.fdt.getprop(self.node.Offset(), 'empty') - self.assertEqual(val, data) + self.assertEqual(tools.ToBytes(val), data)

- self.node.SetData('empty', '123') - self.assertEqual('123', prop.bytes) + self.node.SetData('empty', b'123') + self.assertEqual(b'123', prop.bytes)

def testFromData(self): dtb2 = fdt.Fdt.FromData(self.dtb.GetContents()) @@ -508,7 +508,7 @@ class TestFdtUtil(unittest.TestCase): self.assertEqual(dtb, fdt_util.EnsureCompiled(dtb))

def testGetPlainBytes(self): - self.assertEqual('fred', fdt_util.get_plain_bytes('fred')) + self.assertEqual(b'fred', fdt_util.get_plain_bytes('fred'))

def RunTestCoverage(): diff --git a/tools/patman/tools.py b/tools/patman/tools.py index 976670ef006..bdc1953936c 100644 --- a/tools/patman/tools.py +++ b/tools/patman/tools.py @@ -317,3 +317,28 @@ def ToChar(byte): byte: A byte or str value """ return chr(byte) if type(byte) != str else byte + +def ToChars(byte_list): + """Convert a list of bytes to a str/bytes type + + Args: + byte_list: List of ASCII values representing the string + + Returns: + string made by concatenating all the ASCII values + """ + return ''.join([chr(byte) for byte in byte_list]) + +def ToBytes(string): + """Convert a str type into a bytes type + + Args: + string: string to convert value + + Returns: + Python 3: A bytes type + Python 2: A string type + """ + if sys.version_info[0] >= 3: + return string.encode('utf-8') + return string

Update this class to work correctly on Python 3 and to pass its unit tests. The only required change is to deal with a difference in the behaviour of sorting with a None value.

Signed-off-by: Simon Glass sjg@chromium.org ---

tools/dtoc/fdt.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt.py index f051ce67632..26e683c2a4f 100644 --- a/tools/dtoc/fdt.py +++ b/tools/dtoc/fdt.py @@ -465,8 +465,11 @@ class Node:

# Sync properties now, whose offsets should not have been disturbed. # We do this after subnodes, since this disturbs the offsets of these - # properties. - prop_list = sorted(self.props.values(), key=lambda prop: prop._offset, + # properties. Note that new properties will have an offset of None here, + # which Python 3 cannot sort against int. So use a large value instead + # to ensure that the new properties are added first. + prop_list = sorted(self.props.values(), + key=lambda prop: prop._offset or 1 << 31, reverse=True) for prop in prop_list: prop.Sync(auto_resize)

At present this test does not check the upper 32 bits of the returned value. Add some additional tests to cover this.

Signed-off-by: Simon Glass sjg@chromium.org ---

tools/dtoc/test_fdt.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/tools/dtoc/test_fdt.py b/tools/dtoc/test_fdt.py index 4c39f9a3e28..a5234ce1e88 100755 --- a/tools/dtoc/test_fdt.py +++ b/tools/dtoc/test_fdt.py @@ -498,10 +498,17 @@ class TestFdtUtil(unittest.TestCase): self.assertEqual(2, fdt_util.fdt_cells_to_cpu(val, 1))

dtb2 = fdt.FdtScan('tools/dtoc/dtoc_test_addr64.dts') - node2 = dtb2.GetNode('/test1') - val = node2.props['reg'].value + node1 = dtb2.GetNode('/test1') + val = node1.props['reg'].value self.assertEqual(0x1234, fdt_util.fdt_cells_to_cpu(val, 2))

+ node2 = dtb2.GetNode('/test2') + val = node2.props['reg'].value + self.assertEqual(0x1234567890123456, fdt_util.fdt_cells_to_cpu(val, 2)) + self.assertEqual(0x9876543210987654, fdt_util.fdt_cells_to_cpu(val[2:], + 2)) + self.assertEqual(0x12345678, fdt_util.fdt_cells_to_cpu(val, 1)) + def testEnsureCompiled(self): """Test a degenerate case of this function""" dtb = fdt_util.EnsureCompiled('tools/dtoc/dtoc_test_simple.dts')

Since we are now using the bytes type in Python 3, the conversion in fdt32_to_cpu() is not necessary, so drop it.

Also use 'int' instead of 'long' to convert the integer value, since 'long' is not present in Python 3.

With this, test_fdt passes with both Python 2 and 3:

PYTHONPATH=/tmp/b/sandbox_spl/scripts/dtc/pylibfdt python \ ./tools/dtoc/test_fdt -t

PYTHONPATH=~/cosarm/dtc/pylibfdt:tools/patman python3 \ ./tools/dtoc/test_fdt -t

Signed-off-by: Simon Glass sjg@chromium.org ---

tools/dtoc/fdt_util.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-)

diff --git a/tools/dtoc/fdt_util.py b/tools/dtoc/fdt_util.py index 5fbfc8877bd..f47879ac006 100644 --- a/tools/dtoc/fdt_util.py +++ b/tools/dtoc/fdt_util.py @@ -16,14 +16,6 @@ import tempfile import command import tools

-VERSION3 = sys.version_info > (3, 0) - -def get_plain_bytes(val): - """Handle Python 3 strings""" - if isinstance(val, bytes): - val = val.decode('utf-8') - return val.encode('raw_unicode_escape') - def fdt32_to_cpu(val): """Convert a device tree cell to an integer

@@ -33,9 +25,6 @@ def fdt32_to_cpu(val): Return: A native-endian integer value """ - if VERSION3: - # This code is not reached in Python 2 - val = get_plain_bytes(val) # pragma: no cover return struct.unpack('>I', val)[0]

def fdt_cells_to_cpu(val, cells): @@ -45,11 +34,11 @@ def fdt_cells_to_cpu(val, cells): Value to convert (array of one or more 4-character strings)

Return: - A native-endian long value + A native-endian integer value """ if not cells: return 0 - out = long(fdt32_to_cpu(val[0])) + out = int(fdt32_to_cpu(val[0])) if cells == 2: out = out << 32 | fdt32_to_cpu(val[1]) return out

While reading files in binary mode is the norm, sometimes we want to use text mode. Add an optional parameter to handle this.

Signed-off-by: Simon Glass sjg@chromium.org ---

tools/patman/tools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/patman/tools.py b/tools/patman/tools.py index bdc1953936c..8e9f22afe8a 100644 --- a/tools/patman/tools.py +++ b/tools/patman/tools.py @@ -214,7 +214,7 @@ def Filename(fname): # If not found, just return the standard, unchanged path return fname

-def ReadFile(fname): +def ReadFile(fname, binary=True): """Read and return the contents of a file.

Args: @@ -223,7 +223,7 @@ def ReadFile(fname): Returns: data read from file, as a string. """ - with open(Filename(fname), 'rb') as fd: + with open(Filename(fname), binary and 'rb' or 'r') as fd: data = fd.read() #self._out.Info("Read file '%s' size %d (%#0x)" % #(fname, len(data), len(data)))

With Python 3 we want to use the 'bytes' type instead of 'str'. Adjust the code accordingly so that it works on both Python 2 and Python 3.

Signed-off-by: Simon Glass sjg@chromium.org ---

tools/binman/elf_test.py | 5 +- tools/binman/etype/_testing.py | 2 +- tools/binman/etype/u_boot_dtb_with_ucode.py | 4 +- tools/binman/etype/u_boot_ucode.py | 4 +- tools/binman/etype/vblock.py | 2 +- tools/binman/ftest.py | 132 ++++++++++---------- 6 files changed, 77 insertions(+), 72 deletions(-)

diff --git a/tools/binman/elf_test.py b/tools/binman/elf_test.py index 3d55cb1946e..42d94cbbbe2 100644 --- a/tools/binman/elf_test.py +++ b/tools/binman/elf_test.py @@ -22,7 +22,7 @@ class FakeEntry: """ def __init__(self, contents_size): self.contents_size = contents_size - self.data = 'a' * contents_size + self.data = tools.GetBytes(ord('a'), contents_size)

def GetPath(self): return 'entry_path' @@ -122,7 +122,8 @@ class TestElf(unittest.TestCase): section = FakeSection(sym_value=None) elf_fname = os.path.join(binman_dir, 'test', 'u_boot_binman_syms') syms = elf.LookupAndWriteSymbols(elf_fname, entry, section) - self.assertEqual(tools.GetBytes(255, 16) + 'a' * 4, entry.data) + self.assertEqual(tools.GetBytes(255, 16) + tools.GetBytes(ord('a'), 4), + entry.data)

def testDebug(self): """Check that enabling debug in the elf module produced debug output""" diff --git a/tools/binman/etype/_testing.py b/tools/binman/etype/_testing.py index 3e345bd9526..ac62d2e2005 100644 --- a/tools/binman/etype/_testing.py +++ b/tools/binman/etype/_testing.py @@ -75,7 +75,7 @@ class Entry__testing(Entry): def ObtainContents(self): if self.return_unknown_contents or not self.return_contents: return False - self.data = 'a' + self.data = b'a' self.contents_size = len(self.data) if self.return_contents_once: self.return_contents = False diff --git a/tools/binman/etype/u_boot_dtb_with_ucode.py b/tools/binman/etype/u_boot_dtb_with_ucode.py index 444c51b8b72..188888e022b 100644 --- a/tools/binman/etype/u_boot_dtb_with_ucode.py +++ b/tools/binman/etype/u_boot_dtb_with_ucode.py @@ -26,7 +26,7 @@ class Entry_u_boot_dtb_with_ucode(Entry_blob_dtb): """ def __init__(self, section, etype, node): Entry_blob_dtb.__init__(self, section, etype, node) - self.ucode_data = '' + self.ucode_data = b'' self.collate = False self.ucode_offset = None self.ucode_size = None @@ -65,7 +65,7 @@ class Entry_u_boot_dtb_with_ucode(Entry_blob_dtb): for node in self.ucode.subnodes: data_prop = node.props.get('data') if data_prop: - self.ucode_data += ''.join(data_prop.bytes) + self.ucode_data += data_prop.bytes if self.collate: node.DeleteProp('data') return True diff --git a/tools/binman/etype/u_boot_ucode.py b/tools/binman/etype/u_boot_ucode.py index a00e530295b..dee8848db7a 100644 --- a/tools/binman/etype/u_boot_ucode.py +++ b/tools/binman/etype/u_boot_ucode.py @@ -69,7 +69,7 @@ class Entry_u_boot_ucode(Entry_blob): if entry and entry.target_offset: found = True if not found: - self.data = '' + self.data = b'' return True # Get the microcode from the device tree entry. If it is not available # yet, return False so we will be called later. If the section simply @@ -87,7 +87,7 @@ class Entry_u_boot_ucode(Entry_blob):

if not fdt_entry.collate: # This binary can be empty - self.data = '' + self.data = b'' return True

# Write it out to a file diff --git a/tools/binman/etype/vblock.py b/tools/binman/etype/vblock.py index 334ff9f966a..91fa2f7808f 100644 --- a/tools/binman/etype/vblock.py +++ b/tools/binman/etype/vblock.py @@ -51,7 +51,7 @@ class Entry_vblock(Entry):

def ObtainContents(self): # Join up the data files to be signed - input_data = '' + input_data = b'' for entry_phandle in self.content: data = self.section.GetContentsByPhandle(entry_phandle, self) if data is None: diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 971fade3432..48fec501790 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -29,38 +29,38 @@ import tools import tout

# Contents of test files, corresponding to different entry types -U_BOOT_DATA = '1234' -U_BOOT_IMG_DATA = 'img' -U_BOOT_SPL_DATA = '56780123456789abcde' -U_BOOT_TPL_DATA = 'tpl' -BLOB_DATA = '89' -ME_DATA = '0abcd' -VGA_DATA = 'vga' -U_BOOT_DTB_DATA = 'udtb' -U_BOOT_SPL_DTB_DATA = 'spldtb' -U_BOOT_TPL_DTB_DATA = 'tpldtb' -X86_START16_DATA = 'start16' -X86_START16_SPL_DATA = 'start16spl' -X86_START16_TPL_DATA = 'start16tpl' -PPC_MPC85XX_BR_DATA = 'ppcmpc85xxbr' -U_BOOT_NODTB_DATA = 'nodtb with microcode pointer somewhere in here' -U_BOOT_SPL_NODTB_DATA = 'splnodtb with microcode pointer somewhere in here' -U_BOOT_TPL_NODTB_DATA = 'tplnodtb with microcode pointer somewhere in here' -FSP_DATA = 'fsp' -CMC_DATA = 'cmc' -VBT_DATA = 'vbt' -MRC_DATA = 'mrc' +U_BOOT_DATA = b'1234' +U_BOOT_IMG_DATA = b'img' +U_BOOT_SPL_DATA = b'56780123456789abcde' +U_BOOT_TPL_DATA = b'tpl' +BLOB_DATA = b'89' +ME_DATA = b'0abcd' +VGA_DATA = b'vga' +U_BOOT_DTB_DATA = b'udtb' +U_BOOT_SPL_DTB_DATA = b'spldtb' +U_BOOT_TPL_DTB_DATA = b'tpldtb' +X86_START16_DATA = b'start16' +X86_START16_SPL_DATA = b'start16spl' +X86_START16_TPL_DATA = b'start16tpl' +PPC_MPC85XX_BR_DATA = b'ppcmpc85xxbr' +U_BOOT_NODTB_DATA = b'nodtb with microcode pointer somewhere in here' +U_BOOT_SPL_NODTB_DATA = b'splnodtb with microcode pointer somewhere in here' +U_BOOT_TPL_NODTB_DATA = b'tplnodtb with microcode pointer somewhere in here' +FSP_DATA = b'fsp' +CMC_DATA = b'cmc' +VBT_DATA = b'vbt' +MRC_DATA = b'mrc' TEXT_DATA = 'text' TEXT_DATA2 = 'text2' TEXT_DATA3 = 'text3' -CROS_EC_RW_DATA = 'ecrw' -GBB_DATA = 'gbbd' -BMPBLK_DATA = 'bmp' -VBLOCK_DATA = 'vblk' -FILES_DATA = ("sorry I'm late\nOh, don't bother apologising, I'm " + - "sorry you're alive\n") -COMPRESS_DATA = 'data to compress' -REFCODE_DATA = 'refcode' +CROS_EC_RW_DATA = b'ecrw' +GBB_DATA = b'gbbd' +BMPBLK_DATA = b'bmp' +VBLOCK_DATA = b'vblk' +FILES_DATA = (b"sorry I'm late\nOh, don't bother apologising, I'm " + + b"sorry you're alive\n") +COMPRESS_DATA = b'data to compress' +REFCODE_DATA = b'refcode'

class TestFunctional(unittest.TestCase): @@ -803,7 +803,7 @@ class TestFunctional(unittest.TestCase):

def testPackX86RomMeNoDesc(self): """Test that an invalid Intel descriptor entry is detected""" - TestFunctional._MakeInputFile('descriptor.bin', '') + TestFunctional._MakeInputFile('descriptor.bin', b'') with self.assertRaises(ValueError) as e: self._DoTestFile('031_x86-rom-me.dts') self.assertIn("Node '/binman/intel-descriptor': Cannot find FD " @@ -900,8 +900,8 @@ class TestFunctional(unittest.TestCase): """ first, pos_and_size = self._RunMicrocodeTest('034_x86_ucode.dts', U_BOOT_NODTB_DATA) - self.assertEqual('nodtb with microcode' + pos_and_size + - ' somewhere in here', first) + self.assertEqual(b'nodtb with microcode' + pos_and_size + + b' somewhere in here', first)

def _RunPackUbootSingleMicrocode(self): """Test that x86 microcode can be handled correctly @@ -932,8 +932,8 @@ class TestFunctional(unittest.TestCase): pos_and_size = struct.pack('<2L', 0xfffffe00 + ucode_pos, len(ucode_data)) first = data[:len(U_BOOT_NODTB_DATA)] - self.assertEqual('nodtb with microcode' + pos_and_size + - ' somewhere in here', first) + self.assertEqual(b'nodtb with microcode' + pos_and_size + + b' somewhere in here', first)

def testPackUbootSingleMicrocode(self): """Test that x86 microcode can be handled correctly with fdt_normal. @@ -1068,8 +1068,8 @@ class TestFunctional(unittest.TestCase): self._SetupSplElf('u_boot_ucode_ptr') first, pos_and_size = self._RunMicrocodeTest(dts, U_BOOT_SPL_NODTB_DATA, ucode_second=ucode_second) - self.assertEqual('splnodtb with microc' + pos_and_size + - 'ter somewhere in here', first) + self.assertEqual(b'splnodtb with microc' + pos_and_size + + b'ter somewhere in here', first)

def testPackUbootSplMicrocode(self): """Test that x86 microcode can be handled correctly in SPL""" @@ -1123,8 +1123,9 @@ class TestFunctional(unittest.TestCase): def testSections(self): """Basic test of sections""" data = self._DoReadFile('055_sections.dts') - expected = (U_BOOT_DATA + '!' * 12 + U_BOOT_DATA + 'a' * 12 + - U_BOOT_DATA + '&' * 4) + expected = (U_BOOT_DATA + tools.GetBytes(ord('!'), 12) + + U_BOOT_DATA + tools.GetBytes(ord('a'), 12) + + U_BOOT_DATA + tools.GetBytes(ord('&'), 4)) self.assertEqual(expected, data)

def testMap(self): @@ -1282,8 +1283,10 @@ class TestFunctional(unittest.TestCase): } data, _, _, _ = self._DoReadFileDtb('066_text.dts', entry_args=entry_args) - expected = (TEXT_DATA + tools.GetBytes(0, 8 - len(TEXT_DATA)) + - TEXT_DATA2 + TEXT_DATA3 + 'some text') + expected = (tools.ToBytes(TEXT_DATA) + + tools.GetBytes(0, 8 - len(TEXT_DATA)) + + tools.ToBytes(TEXT_DATA2) + tools.ToBytes(TEXT_DATA3) + + b'some text') self.assertEqual(expected, data)

def testEntryDocs(self): @@ -1304,32 +1307,33 @@ class TestFunctional(unittest.TestCase): """Basic test of generation of a flashrom fmap""" data = self._DoReadFile('067_fmap.dts') fhdr, fentries = fmap_util.DecodeFmap(data[32:]) - expected = U_BOOT_DATA + '!' * 12 + U_BOOT_DATA + 'a' * 12 + expected = (U_BOOT_DATA + tools.GetBytes(ord('!'), 12) + + U_BOOT_DATA + tools.GetBytes(ord('a'), 12)) self.assertEqual(expected, data[:32]) - self.assertEqual('__FMAP__', fhdr.signature) + self.assertEqual(b'__FMAP__', fhdr.signature) self.assertEqual(1, fhdr.ver_major) self.assertEqual(0, fhdr.ver_minor) self.assertEqual(0, fhdr.base) self.assertEqual(16 + 16 + fmap_util.FMAP_HEADER_LEN + fmap_util.FMAP_AREA_LEN * 3, fhdr.image_size) - self.assertEqual('FMAP', fhdr.name) + self.assertEqual(b'FMAP', fhdr.name) self.assertEqual(3, fhdr.nareas) for fentry in fentries: self.assertEqual(0, fentry.flags)

self.assertEqual(0, fentries[0].offset) self.assertEqual(4, fentries[0].size) - self.assertEqual('RO_U_BOOT', fentries[0].name) + self.assertEqual(b'RO_U_BOOT', fentries[0].name)

self.assertEqual(16, fentries[1].offset) self.assertEqual(4, fentries[1].size) - self.assertEqual('RW_U_BOOT', fentries[1].name) + self.assertEqual(b'RW_U_BOOT', fentries[1].name)

self.assertEqual(32, fentries[2].offset) self.assertEqual(fmap_util.FMAP_HEADER_LEN + fmap_util.FMAP_AREA_LEN * 3, fentries[2].size) - self.assertEqual('FMAP', fentries[2].name) + self.assertEqual(b'FMAP', fentries[2].name)

def testBlobNamedByArg(self): """Test we can add a blob with the filename coming from an entry arg""" @@ -1597,7 +1601,7 @@ class TestFunctional(unittest.TestCase): files = entries['files'] entries = files._section._entries

- orig = '' + orig = b'' for i in range(1, 3): key = '%d.dat' % i start = entries[key].image_pos @@ -1625,10 +1629,10 @@ class TestFunctional(unittest.TestCase): """Test an expanding entry""" data, _, map_data, _ = self._DoReadFileDtb('088_expand_size.dts', map=True) - expect = ('a' * 8 + U_BOOT_DATA + - MRC_DATA + 'b' * 1 + U_BOOT_DATA + - 'c' * 8 + U_BOOT_DATA + - 'd' * 8) + expect = (tools.GetBytes(ord('a'), 8) + U_BOOT_DATA + + MRC_DATA + tools.GetBytes(ord('b'), 1) + U_BOOT_DATA + + tools.GetBytes(ord('c'), 8) + U_BOOT_DATA + + tools.GetBytes(ord('d'), 8)) self.assertEqual(expect, data) self.assertEqual('''ImagePos Offset Size Name 00000000 00000000 00000028 main-section @@ -1660,7 +1664,7 @@ class TestFunctional(unittest.TestCase): hash_node = dtb.GetNode('/binman/u-boot/hash').props['value'] m = hashlib.sha256() m.update(U_BOOT_DATA) - self.assertEqual(m.digest(), ''.join(hash_node.value)) + self.assertEqual(m.digest(), b''.join(hash_node.value))

def testHashNoAlgo(self): with self.assertRaises(ValueError) as e: @@ -1683,8 +1687,8 @@ class TestFunctional(unittest.TestCase): hash_node = dtb.GetNode('/binman/section/hash').props['value'] m = hashlib.sha256() m.update(U_BOOT_DATA) - m.update(16 * 'a') - self.assertEqual(m.digest(), ''.join(hash_node.value)) + m.update(tools.GetBytes(ord('a'), 16)) + self.assertEqual(m.digest(), b''.join(hash_node.value))

def testPackUBootTplMicrocode(self): """Test that x86 microcode can be handled correctly in TPL @@ -1699,14 +1703,14 @@ class TestFunctional(unittest.TestCase): TestFunctional._MakeInputFile('tpl/u-boot-tpl', fd.read()) first, pos_and_size = self._RunMicrocodeTest('093_x86_tpl_ucode.dts', U_BOOT_TPL_NODTB_DATA) - self.assertEqual('tplnodtb with microc' + pos_and_size + - 'ter somewhere in here', first) + self.assertEqual(b'tplnodtb with microc' + pos_and_size + + b'ter somewhere in here', first)

def testFmapX86(self): """Basic test of generation of a flashrom fmap""" data = self._DoReadFile('094_fmap_x86.dts') fhdr, fentries = fmap_util.DecodeFmap(data[32:]) - expected = U_BOOT_DATA + MRC_DATA + 'a' * (32 - 7) + expected = U_BOOT_DATA + MRC_DATA + tools.GetBytes(ord('a'), 32 - 7) self.assertEqual(expected, data[:32]) fhdr, fentries = fmap_util.DecodeFmap(data[32:])

@@ -1714,21 +1718,21 @@ class TestFunctional(unittest.TestCase):

self.assertEqual(0, fentries[0].offset) self.assertEqual(4, fentries[0].size) - self.assertEqual('U_BOOT', fentries[0].name) + self.assertEqual(b'U_BOOT', fentries[0].name)

self.assertEqual(4, fentries[1].offset) self.assertEqual(3, fentries[1].size) - self.assertEqual('INTEL_MRC', fentries[1].name) + self.assertEqual(b'INTEL_MRC', fentries[1].name)

self.assertEqual(32, fentries[2].offset) self.assertEqual(fmap_util.FMAP_HEADER_LEN + fmap_util.FMAP_AREA_LEN * 3, fentries[2].size) - self.assertEqual('FMAP', fentries[2].name) + self.assertEqual(b'FMAP', fentries[2].name)

def testFmapX86Section(self): """Basic test of generation of a flashrom fmap""" data = self._DoReadFile('095_fmap_x86_section.dts') - expected = U_BOOT_DATA + MRC_DATA + 'b' * (32 - 7) + expected = U_BOOT_DATA + MRC_DATA + tools.GetBytes(ord('b'), 32 - 7) self.assertEqual(expected, data[:32]) fhdr, fentries = fmap_util.DecodeFmap(data[36:])

@@ -1736,16 +1740,16 @@ class TestFunctional(unittest.TestCase):

self.assertEqual(0, fentries[0].offset) self.assertEqual(4, fentries[0].size) - self.assertEqual('U_BOOT', fentries[0].name) + self.assertEqual(b'U_BOOT', fentries[0].name)

self.assertEqual(4, fentries[1].offset) self.assertEqual(3, fentries[1].size) - self.assertEqual('INTEL_MRC', fentries[1].name) + self.assertEqual(b'INTEL_MRC', fentries[1].name)

self.assertEqual(36, fentries[2].offset) self.assertEqual(fmap_util.FMAP_HEADER_LEN + fmap_util.FMAP_AREA_LEN * 3, fentries[2].size) - self.assertEqual('FMAP', fentries[2].name) + self.assertEqual(b'FMAP', fentries[2].name)

def testElf(self): """Basic test of ELF entries"""

This needs special care to ensure that the bytes type is used for binary data. Add conversion code to deal with strings and bytes correctly.

Signed-off-by: Simon Glass sjg@chromium.org ---

tools/binman/etype/fmap.py | 3 ++- tools/binman/fmap_util.py | 12 +++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/tools/binman/etype/fmap.py b/tools/binman/etype/fmap.py index bf35a5bbf4e..e6b5c5c74c0 100644 --- a/tools/binman/etype/fmap.py +++ b/tools/binman/etype/fmap.py @@ -7,6 +7,7 @@

from entry import Entry import fmap_util +import tools

class Entry_fmap(Entry): @@ -46,7 +47,7 @@ class Entry_fmap(Entry): if pos is not None: pos -= entry.section.GetRootSkipAtStart() areas.append(fmap_util.FmapArea(pos or 0, entry.size or 0, - entry.name, 0)) + tools.FromUnicode(entry.name), 0))

entries = self.section._image.GetEntries() areas = [] diff --git a/tools/binman/fmap_util.py b/tools/binman/fmap_util.py index be3cbee87bd..d0f956b6221 100644 --- a/tools/binman/fmap_util.py +++ b/tools/binman/fmap_util.py @@ -8,9 +8,12 @@

import collections import struct +import sys + +import tools

# constants imported from lib/fmap.h -FMAP_SIGNATURE = '__FMAP__' +FMAP_SIGNATURE = b'__FMAP__' FMAP_VER_MAJOR = 1 FMAP_VER_MINOR = 0 FMAP_STRLEN = 32 @@ -50,6 +53,8 @@ FmapArea = collections.namedtuple('FmapArea', FMAP_AREA_NAMES)

def NameToFmap(name): + if type(name) == bytes and sys.version_info[0] >= 3: + name = name.decode('utf-8') # pragma: no cover (for Python 2) return name.replace('\0', '').replace('-', '_').upper()

def ConvertName(field_names, fields): @@ -65,7 +70,7 @@ def ConvertName(field_names, fields): value: value of that field (string for the ones we support) """ name_index = field_names.index('name') - fields[name_index] = NameToFmap(fields[name_index]) + fields[name_index] = tools.ToBytes(NameToFmap(fields[name_index]))

def DecodeFmap(data): """Decode a flashmap into a header and list of areas @@ -106,7 +111,8 @@ def EncodeFmap(image_size, name, areas): ConvertName(names, params) return struct.pack(fmt, *params)

- values = FmapHeader(FMAP_SIGNATURE, 1, 0, 0, image_size, name, len(areas)) + values = FmapHeader(FMAP_SIGNATURE, 1, 0, 0, image_size, + tools.FromUnicode(name), len(areas)) blob = _FormatBlob(FMAP_HEADER_FORMAT, FMAP_HEADER_NAMES, values) for area in areas: blob += _FormatBlob(FMAP_AREA_FORMAT, FMAP_AREA_NAMES, area)

Add the missing 's' to the required '%s' here.

Signed-off-by: Simon Glass sjg@chromium.org ---

tools/binman/ftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 48fec501790..d0a8b751a2c 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -395,7 +395,7 @@ class TestFunctional(unittest.TestCase): for grep in grep_list: if grep in target: return - self.fail("Error: '%' not found in '%s'" % (grep_list, target)) + self.fail("Error: '%s' not found in '%s'" % (grep_list, target))

def CheckNoGaps(self, entries): """Check that all entries fit together without gaps

Since binman can run tests in parallel, document this.

Signed-off-by: Simon Glass sjg@chromium.org ---

tools/binman/README | 14 ++++++++++++++ 1 file changed, 14 insertions(+)

diff --git a/tools/binman/README b/tools/binman/README index 927fa856acf..ac193f16cf7 100644 --- a/tools/binman/README +++ b/tools/binman/README @@ -702,6 +702,20 @@ To enable Python test coverage on Debian-type distributions (e.g. Ubuntu): $ sudo apt-get install python-coverage python-pytest

+Concurrent tests +---------------- + +Binman tries to run tests concurrently. This means that the tests make use of +all available CPUs to run. + + To enable this: + + $ sudo apt-get install python-subunit python3-subunit + +Use '-P 1' to disable this. It is automatically disabled when code coverage is +being used (-T) since they are incompatible. + + Advanced Features / Technical docs ----------------------------------

We need slightly different commands to run code coverage with Python 3. Update the RunTestCoverage() function to handle this.

Signed-off-by: Simon Glass sjg@chromium.org ---

tools/patman/test_util.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/tools/patman/test_util.py b/tools/patman/test_util.py index e462ec8f72b..ea36cd16339 100644 --- a/tools/patman/test_util.py +++ b/tools/patman/test_util.py @@ -17,6 +17,8 @@ try: except ImportError: from io import StringIO

+PYTHON = 'python%d' % sys.version_info[0] +

def RunTestCoverage(prog, filter_fname, exclude_list, build_dir, required=None): """Run tests and check that we get 100% coverage @@ -43,11 +45,12 @@ def RunTestCoverage(prog, filter_fname, exclude_list, build_dir, required=None): else: glob_list = [] glob_list += exclude_list - glob_list += ['*libfdt.py', '*site-packages*'] - cmd = ('PYTHONPATH=$PYTHONPATH:%s/sandbox_spl/tools python-coverage run ' - '--omit "%s" %s -P1 -t' % (build_dir, ','.join(glob_list), prog)) + glob_list += ['*libfdt.py', '*site-packages*', '*dist-packages*'] + cmd = ('PYTHONPATH=$PYTHONPATH:%s/sandbox_spl/tools %s-coverage run ' + '--omit "%s" %s -P1 -t' % (build_dir, PYTHON, ','.join(glob_list), + prog)) os.system(cmd) - stdout = command.Output('python-coverage', 'report') + stdout = command.Output('%s-coverage' % PYTHON, 'report') lines = stdout.splitlines() if required: # Convert '/path/to/name.py' just the module name 'name' @@ -65,8 +68,8 @@ def RunTestCoverage(prog, filter_fname, exclude_list, build_dir, required=None): print(coverage) if coverage != '100%': print(stdout) - print("Type 'python-coverage html' to get a report in " - 'htmlcov/index.html') + print("Type '%s-coverage html' to get a report in " + 'htmlcov/index.html' % PYTHON) print('Coverage error: %s, but should be 100%%' % coverage) ok = False if not ok: