Re: [U-Boot] [PATCH 2/2] arm64: add better spin-table support

On 06/17/2016 05:51 AM, Masahiro Yamada wrote:

There are two enable methods supported by ARM64 Linux; psci and spin-table. The latter is simpler and easier to use for quick SoC bring-up.

So, I used the spin-table for my first ARMv8 SoC porting, but I found its support in U-Boot was poor. It is true there exists a code fragment for the spin code in arch/arm/cpu/armv8/start.S, but I see some problems:

• We must hard-code CPU_RELEASE_ADDR so that it matches the "cpu-release-addr" property in the DT that comes from the kernel tree.

• The Documentation/arm64/booting.txt in Linux requires that the release address must be zero-initialized, but it is not cared by the common code in U-Boot. So, we must do it in a board specific manner.

• There is no systematic way to protect the spin code from the kernel. U-Boot relocates itself during the boot, so it is difficult to predict where the spin code will be located after the relocation, which makes it even more difficult to hard-code /memreserve/ in the DT of the kernel. One possible work-around would be to pre-fetch the spin-code into the I-cache of secondary CPUs, but this is an unsafe solution.

So, here is a patch to solve the problems. In this approach, the DT is run-time modified to reserve the spin code (+ cpu-release-addr). Also, the "cpu-release-addr" property is set to an appropriate address after the relocation, which means we no longer need the hard-coded CPU_RELEASE_ADDR.

Currently this patch only supports ARMv8, but theoretically nothing about the spin-table is arch-specific. Perhaps, we might want to support it on PowerPC in the future. So, I put the DT fixup code into the common/ directory. Very little code must be written in assembler, which went to the arch/arm/cpu/armv8/ directory.

Signed-off-by: Masahiro Yamada yamada.masahiro@socionext.com

arch/arm/cpu/armv8/Kconfig | 18 +++++++++++ arch/arm/cpu/armv8/Makefile | 1 + arch/arm/cpu/armv8/spin_table_v8.S | 22 ++++++++++++++ arch/arm/cpu/armv8/start.S | 10 +++--- arch/arm/lib/bootm-fdt.c | 7 +++++ common/Makefile | 1 + common/spin_table.c | 62 ++++++++++++++++++++++++++++++++++++++ include/spin_table.h | 14 +++++++++ 8 files changed, 131 insertions(+), 4 deletions(-) create mode 100644 arch/arm/cpu/armv8/spin_table_v8.S create mode 100644 common/spin_table.c create mode 100644 include/spin_table.h

diff --git a/arch/arm/cpu/armv8/Kconfig b/arch/arm/cpu/armv8/Kconfig index 3d19bbf..019b625 100644 --- a/arch/arm/cpu/armv8/Kconfig +++ b/arch/arm/cpu/armv8/Kconfig @@ -3,4 +3,22 @@ if ARM64 config ARMV8_MULTIENTRY boolean "Enable multiple CPUs to enter into U-Boot"

+config SPIN_TABLE

• bool "Support spin-table enable method"
• depends on ARMV8_MULTIENTRY && OF_LIBFDT
• help

•  Say Y here to support "spin-table" enable method for booting Linux.
  

•  To use this feature, you must do:
  

•    - Specify enable-method = "spin-table" in each CPU node in the
  

•      Device Tree you are using to boot the kernel
  

•    - Let secondary CPUs in U-Boot (in a board specific manner)
  

•      before the master CPU jumps to the kernel
  

•  U-Boot automatically does:
  

•    - Set "cpu-release-addr" property of each CPU node
  

•      (overwrites it if already exists).
  

•    - Reserve the code for the spin-table and the release address
  

•      via a /memreserve/ region in the Device Tree.
  

• endif

diff --git a/arch/arm/cpu/armv8/Makefile b/arch/arm/cpu/armv8/Makefile index 1c85aa9..2e3f421 100644 --- a/arch/arm/cpu/armv8/Makefile +++ b/arch/arm/cpu/armv8/Makefile @@ -15,6 +15,7 @@ obj-y += cache.o obj-y += tlb.o obj-y += transition.o obj-y += fwcall.o +obj-$(CONFIG_SPIN_TABLE) += spin_table_v8.o

obj-$(CONFIG_FSL_LAYERSCAPE) += fsl-layerscape/ obj-$(CONFIG_ARCH_ZYNQMP) += zynqmp/ diff --git a/arch/arm/cpu/armv8/spin_table_v8.S b/arch/arm/cpu/armv8/spin_table_v8.S new file mode 100644 index 0000000..2f1bd61 --- /dev/null +++ b/arch/arm/cpu/armv8/spin_table_v8.S @@ -0,0 +1,22 @@ +/*

• • Copyright (C) 2016 Masahiro Yamada yamada.masahiro@socionext.com
• • SPDX-License-Identifier: GPL-2.0+
• */

+#include <linux/linkage.h>

+ENTRY(spin_table_secondary_jump) +.globl spin_table_reserve_begin +spin_table_reserve_begin: +0: wfe

• ldr x0, spin_table_cpu_release_addr
• cbz x0, 0b
• br x0

+.globl spin_table_cpu_release_addr

• .align 3

+spin_table_cpu_release_addr:

• .quad 0

+.globl spin_table_reserve_end +spin_table_reserve_end: +ENDPROC(spin_table_secondary_jump) diff --git a/arch/arm/cpu/armv8/start.S b/arch/arm/cpu/armv8/start.S index 670e323..15ca864 100644 --- a/arch/arm/cpu/armv8/start.S +++ b/arch/arm/cpu/armv8/start.S @@ -94,7 +94,11 @@ reset: /* Processor specific initialization */ bl lowlevel_init

-#ifdef CONFIG_ARMV8_MULTIENTRY +#if defined(CONFIG_SPIN_TABLE)

• branch_if_master x0, x1, master_cpu
• b spin_table_secondary_jump
• /* never return */

+#elif defined(CONFIG_ARMV8_MULTIENTRY) branch_if_master x0, x1, master_cpu

/* @@ -106,10 +110,8 @@ slave_cpu: ldr x0, [x1] cbz x0, slave_cpu br x0 /* branch to the given address */ -master_cpu:

• /* On the master CPU */ #endif /* CONFIG_ARMV8_MULTIENTRY */

+master_cpu: bl _main

#ifdef CONFIG_SYS_RESET_SCTRL diff --git a/arch/arm/lib/bootm-fdt.c b/arch/arm/lib/bootm-fdt.c index 76b75d8..ec581ac 100644 --- a/arch/arm/lib/bootm-fdt.c +++ b/arch/arm/lib/bootm-fdt.c @@ -21,6 +21,7 @@ #include <asm/armv7.h> #endif #include <asm/psci.h> +#include <spin_table.h>

DECLARE_GLOBAL_DATA_PTR;

@@ -51,5 +52,11 @@ int arch_fixup_fdt(void *blob) return ret; #endif

+#ifdef CONFIG_SPIN_TABLE

• ret = spin_table_update_dt(blob);
• if (ret)

• return ret;
  

+#endif

• return 0; }

diff --git a/common/Makefile b/common/Makefile index 1557a04..f4eb381 100644 --- a/common/Makefile +++ b/common/Makefile @@ -141,6 +141,7 @@ obj-$(CONFIG_ANDROID_BOOT_IMAGE) += image-android.o obj-$(CONFIG_$(SPL_)OF_LIBFDT) += image-fdt.o obj-$(CONFIG_$(SPL_)FIT) += image-fit.o obj-$(CONFIG_$(SPL_)FIT_SIGNATURE) += image-sig.o +obj-$(CONFIG_$(SPL_)SPIN_TABLE) += spin_table.o obj-$(CONFIG_IO_TRACE) += iotrace.o obj-y += memsize.o obj-y += stdio.o diff --git a/common/spin_table.c b/common/spin_table.c new file mode 100644 index 0000000..046af8f --- /dev/null +++ b/common/spin_table.c @@ -0,0 +1,62 @@ +/*

• • Copyright (C) 2016 Masahiro Yamada yamada.masahiro@socionext.com
• • SPDX-License-Identifier: GPL-2.0+
• */

+#include <common.h> +#include <libfdt.h> +#include <spin_table.h>

+int spin_table_update_dt(void *fdt) +{

• int cpus_offset, offset;
• const char *prop;
• int ret;
• unsigned long rsv_addr = (unsigned long)&spin_table_reserve_begin;
• unsigned long rsv_size = &spin_table_reserve_end -

• 				&spin_table_reserve_begin;
  

• cpus_offset = fdt_path_offset(fdt, "/cpus");
• if (cpus_offset < 0)

• return -ENODEV;
  

• for (offset = fdt_first_subnode(fdt, cpus_offset);

•     offset >= 0;
  

•     offset = fdt_next_subnode(fdt, offset)) {
  

• prop = fdt_getprop(fdt, offset, "device_type", NULL);
  

• if (!prop || strcmp(prop, "cpu"))
  

• 	continue;
  

• /*
  

•  * In the first loop, we check if every CPU node specifies
  

•  * spin-table.  Otherwise, just return successfully to not
  

•  * disturb other methods, like psci.
  

•  */
  

• prop = fdt_getprop(fdt, offset, "enable-method", NULL);
  

• if (!prop || strcmp(prop, "spin-table"))
  

• 	return 0;
  

• }
• for (offset = fdt_first_subnode(fdt, cpus_offset);

•     offset >= 0;
  

•     offset = fdt_next_subnode(fdt, offset)) {
  

• prop = fdt_getprop(fdt, offset, "device_type", NULL);
  

• if (!prop || strcmp(prop, "cpu"))
  

• 	continue;
  

• ret = fdt_setprop_u64(fdt, offset, "cpu-release-addr",
  

• 		(unsigned long)&spin_table_cpu_release_addr);
  

• if (ret)
  

• 	return -ENOSPC;
  

• }
• ret = fdt_add_mem_rsv(fdt, rsv_addr, rsv_size);
• if (ret)

• return -ENOSPC;
  

• printf(" Reserved memory region for spin-table: addr=%lx size=%lx\n",

•       rsv_addr, rsv_size);
  

• return 0;

+} diff --git a/include/spin_table.h b/include/spin_table.h new file mode 100644 index 0000000..cf455a1 --- /dev/null +++ b/include/spin_table.h @@ -0,0 +1,14 @@ +/*

• • SPDX-License-Identifier: GPL-2.0+
• */

+#ifndef __SPIN_TABLE_H__ +#define __SPIN_TABLE_H__

+extern u64 spin_table_cpu_release_addr; +extern char spin_table_reserve_begin; +extern char spin_table_reserve_end;

+int spin_table_update_dt(void *fdt);

+#endif /* __SPIN_TABLE_H__ */

Masahiro,

Because ARMv8 had no spin table support at time of our bring-up, and the core id is SoC specific, we put the code under SoC folder, now arch/arm/cpu/armv8/fsl-layerscape/fdt.c and arch/arm/cpu/armv8/fsl-layerscape/lowlevel.S. We actually use multiple tables. More specifically one table per core. The spin table is protected from kernel by fdt_add_mem_rsv() function.

It will be great if we can consolidate the code and reuse what we already have.

York