Re: [U-Boot] [PATCH 5/7] riscv: add support for multi-hart systems

Hi Lukas

...

From: Lukas Auer [mailto:lukas.auer@aisec.fraunhofer.de] Sent: Tuesday, February 12, 2019 6:14 AM To: u-boot@lists.denx.de Cc: Atish Patra; Anup Patel; Bin Meng; Andreas Schwab; Palmer Dabbelt; Alexander Graf; Lukas Auer; Anup Patel; Rick Jian-Zhi Chen(陳建志); Baruch Siach; Stefan Roese Subject: [PATCH 5/7] riscv: add support for multi-hart systems

On RISC-V, all harts boot independently. To be able to run on a multi-hart system, U-Boot must be extended with the functionality to manage all harts in the system. A new config option, CONFIG_MAIN_HART, is used to select the hart U-Boot runs on. All other harts are halted. U-Boot can delegate functions to them using smp_call_function().

Every hart has a valid pointer to the global data structure and a 8KiB stack by default. The stack size is set with CONFIG_STACK_SIZE_SHIFT.

Signed-off-by: Lukas Auer lukas.auer@aisec.fraunhofer.de

arch/riscv/Kconfig | 12 +++++ arch/riscv/cpu/start.S | 102 ++++++++++++++++++++++++++++++++++- arch/riscv/include/asm/csr.h | 1 + 3 files changed, 114 insertions(+), 1 deletion(-)

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 3a51339c4d..af8d0f8d67 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -140,4 +140,16 @@ config SBI_IPI default y if RISCV_SMODE depends on SMP

+config MAIN_HART

• int "Main hart in system"
  

• default 0
  

• help
  

•   Some SoCs include harts of various sizes, some of which might not
  

•   be suitable for running U-Boot. CONFIG_MAIN_HART is used to select
  

•   the hart U-Boot runs on.
  

+config STACK_SIZE_SHIFT

• int
  

• default 13
  

endmenu diff --git a/arch/riscv/cpu/start.S b/arch/riscv/cpu/start.S index a30f6f7194..ce7230df37 100644 --- a/arch/riscv/cpu/start.S +++ b/arch/riscv/cpu/start.S @@ -13,6 +13,7 @@ #include <config.h> #include <common.h> #include <elf.h> +#include <asm/csr.h> #include <asm/encoding.h> #include <generated/asm-offsets.h>

@@ -45,6 +46,23 @@ _start: /* mask all interrupts */ csrw MODE_PREFIX(ie), zero

+#ifdef CONFIG_SMP

• /* check if hart is within range */
  

• /* s0: hart id */
  

• li      t0, CONFIG_NR_CPUS
  

• bge     s0, t0, hart_out_of_bounds_loop
  

+#endif

+#ifdef CONFIG_SMP

• /* set xSIE bit to receive IPIs */
  

+#ifdef CONFIG_RISCV_MMODE

• li      t0, MIE_MSIE
  

+#else

• li      t0, SIE_SSIE
  

+#endif

• csrs    MODE_PREFIX(ie), t0
  

+#endif

/*

• Set stackpointer in internal/ex RAM to call board_init_f

*/ @@ -56,7 +74,25 @@ call_board_init_f: call_board_init_f_0: mv a0, sp jal board_init_f_alloc_reserve

• /*
  

•  * Set global data pointer here for all harts, uninitialized at this
  

•  * point.
  

•  */
  

• mv      gp, a0
  

• /* setup stack */
  

+#ifdef CONFIG_SMP

• /* s0: hart id */
  

• slli    t0, s0, CONFIG_STACK_SIZE_SHIFT
  

• sub     sp, a0, t0
  

+#else mv sp, a0 +#endif

• /* Continue on main hart, others branch to secondary_hart_loop */
  

• li      t0, CONFIG_MAIN_HART
  

• bne     s0, t0, secondary_hart_loop
  
  la      t0, prior_stage_fdt_address
  SREG    s1, 0(t0)
  

@@ -95,7 +131,14 @@ relocate_code: *Set up the stack */ stack_setup: +#ifdef CONFIG_SMP

• /* s0: hart id */
  

• slli    t0, s0, CONFIG_STACK_SIZE_SHIFT
  

• sub     sp, s2, t0
  

+#else mv sp, s2 +#endif

• la      t0, _start
  sub     t6, s4, t0              /* t6 <- relocation offset */
  beq     t0, s4, clear_bss       /* skip relocation */
  

@@ -175,13 +218,30 @@ clear_bss: add t0, t0, t6 /* t0 <- rel __bss_start in RAM */ la t1, __bss_end /* t1 <- rel __bss_end in FLASH */ add t1, t1, t6 /* t1 <- rel __bss_end in RAM */

• beq     t0, t1, call_board_init_r
  

• beq     t0, t1, relocate_secondary_harts
  
  

clbss_l: SREG zero, 0(t0) /* clear loop... */ addi t0, t0, REGBYTES bne t0, t1, clbss_l

+relocate_secondary_harts: +#ifdef CONFIG_SMP

• /* send relocation IPI */
  

• la      t0, secondary_hart_relocate
  

• add     a0, t0, t6
  

• /* store relocation offset */
  

• mv      s5, t6
  

• mv      a1, s2
  

• mv      a2, s3
  

• jal     smp_call_function
  

• /* restore relocation offset */
  

• mv      t6, s5
  

+#endif

/*

• We are done. Do not return, instead branch to second part of board
• initialization, now running from RAM.

@@ -202,3 +262,43 @@ call_board_init_r:

• jump to it ...

*/ jr t4 /* jump to board_init_r() */

+#ifdef CONFIG_SMP +hart_out_of_bounds_loop:

• /* Harts in this loop are out of bounds, increase CONFIG_NR_CPUS. */
  

• wfi
  

• j       hart_out_of_bounds_loop
  

+#endif

+#ifdef CONFIG_SMP +/* SMP relocation entry */ +secondary_hart_relocate:

• /* a1: new sp */
  

• /* a2: new gd */
  

• /* s0: hart id */
  

• /* setup stack */
  

• slli    t0, s0, CONFIG_STACK_SIZE_SHIFT
  

• sub     sp, a1, t0
  

• /* update global data pointer */
  

• mv      gp, a2
  

+#endif

+secondary_hart_loop:

• wfi
  

+#ifdef CONFIG_SMP

• csrr    t0, MODE_PREFIX(ip)
  

+#ifdef CONFIG_RISCV_MMODE

• andi    t0, t0, MIE_MSIE
  

+#else

• andi    t0, t0, SIE_SSIE
  

+#endif

• beqz    t0, secondary_hart_loop
  

• mv      a0, s0
  

• jal     handle_ipi
  

I found that s0 maybe corrupted after execute handle_ipi. Because smp_function will be treated as a return function by compiler, so compiler will generate codes to execute restore after smp_function().

But actually it is a no-return function. So there maybe no chance to execute restore. And s0 will be corrupted somehow.

The usage of s0 in v2 flow seem the same as v1. So I reply mail in v1 patch.

Thanks Rick

...

+#endif

• j       secondary_hart_loop
  

diff --git a/arch/riscv/include/asm/csr.h b/arch/riscv/include/asm/csr.h index 86136f542c..644e6baa15 100644 --- a/arch/riscv/include/asm/csr.h +++ b/arch/riscv/include/asm/csr.h @@ -46,6 +46,7 @@ #endif

/* Interrupt Enable and Interrupt Pending flags */ +#define MIE_MSIE _AC(0x00000008, UL) /* Software Interrupt Enable */ #define SIE_SSIE _AC(0x00000002, UL) /* Software Interrupt Enable */ #define SIE_STIE _AC(0x00000020, UL) /* Timer Interrupt Enable */

-- 2.20.1