-----Original Message----- From: Lukas Auer [mailto:lukas.auer@aisec.fraunhofer.de] Sent: Tuesday, February 12, 2019 3:44 AM To: u-boot@lists.denx.de Cc: Atish Patra Atish.Patra@wdc.com; Anup Patel Anup.Patel@wdc.com; Bin Meng bmeng.cn@gmail.com; Andreas Schwab schwab@suse.de; Palmer Dabbelt palmer@sifive.com; Alexander Graf agraf@suse.de; Lukas Auer lukas.auer@aisec.fraunhofer.de; Anup Patel anup@brainfault.org; Rick Chen rick@andestech.com Subject: [PATCH 1/7] riscv: add infrastructure for calling functions on other harts

Harts on RISC-V boot independently and U-Boot is responsible for managing them. Functions are called on other harts with smp_call_function(), which sends inter-processor interrupts (IPIs) to all other harts. Functions are specified with their address and two function arguments (argument 2 and 3). The first function argument is always the hart ID of the hart calling the function. On the other harts, the IPI interrupt handler handle_ipi() must be called on software interrupts to handle the request and call the specified function.

Functions are stored in the ipi_data data structure. Every hart has its own data structure in global data. While this is not required at the moment (all harts are expected to boot Linux), this does allow future expansion, where other harts may be used for monitoring or other tasks.

Signed-off-by: Lukas Auer lukas.auer@aisec.fraunhofer.de

arch/riscv/Kconfig | 19 +++++ arch/riscv/include/asm/global_data.h | 5 ++ arch/riscv/include/asm/smp.h | 53 +++++++++++++ arch/riscv/lib/Makefile | 1 + arch/riscv/lib/smp.c | 110 +++++++++++++++++++++++++++ 5 files changed, 188 insertions(+) create mode 100644 arch/riscv/include/asm/smp.h create mode 100644 arch/riscv/lib/smp.c

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index c45e4d73a8..c0842178dd 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -116,4 +116,23 @@ config RISCV_RDTIME config SYS_MALLOC_F_LEN default 0x1000

+config SMP

• bool "Symmetric Multi-Processing"
• help

•  This enables support for systems with more than one CPU. If
  

•  you say N here, U-Boot will run on single and multiprocessor
  

•  machines, but will use only one CPU of a multiprocessor
  

•  machine. If you say Y here, U-Boot will run on many, but not
  

•  all, single processor machines.
  

+config NR_CPUS

• int "Maximum number of CPUs (2-32)"
• range 2 32
• depends on SMP
• default "8"
• help

•  On multiprocessor machines, U-Boot sets up a stack for each CPU.
  

•  Stack memory is pre-allocated. U-Boot must therefore know the
  

•  maximum number of CPUs that may be present.
  

endmenu diff --git a/arch/riscv/include/asm/global_data.h b/arch/riscv/include/asm/global_data.h index a3a342c6e1..23a5f35af5 100644 --- a/arch/riscv/include/asm/global_data.h +++ b/arch/riscv/include/asm/global_data.h @@ -10,12 +10,17 @@ #ifndef __ASM_GBL_DATA_H #define __ASM_GBL_DATA_H

+#include <asm/smp.h>

/* Architecture-specific global data */ struct arch_global_data { long boot_hart; /* boot hart id */ #ifdef CONFIG_SIFIVE_CLINT void __iomem *clint; /* clint base address */ #endif +#ifdef CONFIG_SMP

• struct ipi_data ipi[CONFIG_NR_CPUS];

+#endif };

#include <asm-generic/global_data.h> diff --git a/arch/riscv/include/asm/smp.h b/arch/riscv/include/asm/smp.h new file mode 100644 index 0000000000..bc863fdbaf --- /dev/null +++ b/arch/riscv/include/asm/smp.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/*

• • Copyright (C) 2019 Fraunhofer AISEC,
• • Lukas Auer lukas.auer@aisec.fraunhofer.de */

+#ifndef _ASM_RISCV_SMP_H +#define _ASM_RISCV_SMP_H

+/**

• • struct ipi_data - Inter-processor interrupt (IPI) data structure
• • IPIs are used for SMP support to communicate to other harts what

+function to

• • call. Functions are in the form
• • void (*addr)(ulong hart, ulong arg0, ulong arg1).
• • The function address and the two arguments, arg0 and arg1, are

+stored in the

• • IPI data structure. The hart ID is inserted by the hart handling the

+IPI and

• • calling the function.
• • @addr: Address of function
• • @arg0: First argument of function
• • @arg1: Second argument of function
• */

+struct ipi_data {

• ulong addr;
• ulong arg0;
• ulong arg1;

+};

+/**

• • handle_ipi() - interrupt handler for software interrupts
• • The IPI interrupt handler must be called to handle software

+interrupts. It

• • calls the function specified in the hart's IPI data structure.
• • @hart: Hart ID of the current hart
• */

+void handle_ipi(ulong hart);

+/**

• • smp_call_function() - Call a function on all other harts
• • Send IPIs with the specified function call to all harts.
• • @addr: Address of function
• • @arg0: First argument of function
• • @arg1: Second argument of function
• • @return 0 if OK, -ve on error
• */

+int smp_call_function(ulong addr, ulong arg0, ulong arg1);

+#endif diff --git a/arch/riscv/lib/Makefile b/arch/riscv/lib/Makefile index edfa61690c..19370f9749 100644 --- a/arch/riscv/lib/Makefile +++ b/arch/riscv/lib/Makefile @@ -14,6 +14,7 @@ obj-$(CONFIG_SIFIVE_CLINT) += sifive_clint.o obj-y += interrupts.o obj-y += reset.o obj-y += setjmp.o +obj-$(CONFIG_SMP) += smp.o

# For building EFI apps CFLAGS_$(EFI_CRT0) := $(CFLAGS_EFI) diff --git a/arch/riscv/lib/smp.c b/arch/riscv/lib/smp.c new file mode 100644 index 0000000000..1266a2a0ef --- /dev/null +++ b/arch/riscv/lib/smp.c @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: GPL-2.0+ +/*

• • Copyright (C) 2019 Fraunhofer AISEC,
• • Lukas Auer lukas.auer@aisec.fraunhofer.de */

+#include <common.h> +#include <dm.h> +#include <asm/barrier.h> +#include <asm/smp.h>

+DECLARE_GLOBAL_DATA_PTR;

+/**

• • riscv_send_ipi() - Send inter-processor interrupt (IPI)
• • Platform code must provide this function.
• • @hart: Hart ID of receiving hart
• • @return 0 if OK, -ve on error
• */

+extern int riscv_send_ipi(int hart);

+/**

• • riscv_clear_ipi() - Clear inter-processor interrupt (IPI)
• • Platform code must provide this function.
• • @hart: Hart ID of hart to be cleared
• • @return 0 if OK, -ve on error
• */

+extern int riscv_clear_ipi(int hart);

+static int send_ipi_many(struct ipi_data *ipi) {

• ofnode node, cpus;
• u32 reg;
• int ret;
• cpus = ofnode_path("/cpus");
• if (!ofnode_valid(cpus)) {

• pr_err("Can't find cpus node!\n");
  

• return -EINVAL;
  

• }
• ofnode_for_each_subnode(node, cpus) {

• if (!ofnode_is_available(node))
  

• 	continue;
  

It is not correct to assume that whatever CPUs are marked available will come online. It is possible that certain available CPUs failed to come online due HW failure.

Better approach would be keep an atomic bitmask of HARTs that have entered U-Boot. All HARTs that enter U-Boot will update the atomic HART available bitmask. We send IPI only to HARTs that are available as-per atomic HART bitmask.

• /* read hart ID of CPU */
  

• ret = ofnode_read_u32(node, "reg", &reg);
  

• if (ret)
  

• 	continue;
  

• /* skip if it is the hart we are running on */
  

• if (reg == gd->arch.boot_hart)
  

• 	continue;
  

• if (reg >= CONFIG_NR_CPUS) {
  

• 	pr_err("Hart ID %d is out of range, increase
  

CONFIG_NR_CPUS\n",

• 	       reg);
  

• 	continue;
  

• }
  

• gd->arch.ipi[reg].addr = ipi->addr;
  

• gd->arch.ipi[reg].arg0 = ipi->arg0;
  

• gd->arch.ipi[reg].arg1 = ipi->arg1;
  

• mb();
  

• ret = riscv_send_ipi(reg);
  

• if (ret)
  

• 	return ret;
  

• }
• return 0;

+}

+void handle_ipi(ulong hart) +{

• int ret;
• void (*smp_function)(ulong hart, ulong arg0, ulong arg1);
• if (hart >= CONFIG_NR_CPUS)

• return;
  

• ret = riscv_clear_ipi(hart);
• if (ret) {

• pr_err("Cannot clear IPI\n");
  

• return;
  

• }
• smp_function = (void (*)(ulong, ulong, ulong))gd-

...

arch.ipi[hart].addr;

• invalidate_icache_all();
• smp_function(hart, gd->arch.ipi[hart].arg0, gd->arch.ipi[hart].arg1);

+}

+int smp_call_function(ulong addr, ulong arg0, ulong arg1) {

• int ret = 0;
• struct ipi_data ipi;
• ipi.addr = addr;
• ipi.arg0 = arg0;
• ipi.arg1 = arg1;
• ret = send_ipi_many(&ipi);
• return ret;

+}

2.20.1

Regards, Anup

Hi Lukas,

On Tue, Feb 12, 2019 at 6:14 AM Lukas Auer lukas.auer@aisec.fraunhofer.de wrote:

Harts on RISC-V boot independently and U-Boot is responsible for managing them. Functions are called on other harts with smp_call_function(), which sends inter-processor interrupts (IPIs) to all other harts. Functions are specified with their address and two function arguments (argument 2 and 3). The first function argument is always the hart ID of the hart calling the function. On the other harts, the IPI interrupt handler handle_ipi() must be called on software interrupts to handle the request and call the specified function.

Functions are stored in the ipi_data data structure. Every hart has its own data structure in global data. While this is not required at the moment (all harts are expected to boot Linux), this does allow future expansion, where other harts may be used for monitoring or other tasks.

Signed-off-by: Lukas Auer lukas.auer@aisec.fraunhofer.de

arch/riscv/Kconfig | 19 +++++ arch/riscv/include/asm/global_data.h | 5 ++ arch/riscv/include/asm/smp.h | 53 +++++++++++++ arch/riscv/lib/Makefile | 1 + arch/riscv/lib/smp.c | 110 +++++++++++++++++++++++++++ 5 files changed, 188 insertions(+) create mode 100644 arch/riscv/include/asm/smp.h create mode 100644 arch/riscv/lib/smp.c

Looks pretty clean to me. Thanks! Some nits below.

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index c45e4d73a8..c0842178dd 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -116,4 +116,23 @@ config RISCV_RDTIME config SYS_MALLOC_F_LEN default 0x1000

+config SMP

•   bool "Symmetric Multi-Processing"
  

•   help
  

•     This enables support for systems with more than one CPU. If
  

•     you say N here, U-Boot will run on single and multiprocessor
  

•     machines, but will use only one CPU of a multiprocessor
  

•     machine. If you say Y here, U-Boot will run on many, but not
  

•     all, single processor machines.
  

U-Boot will run on both single and multiprocessor machines?

+config NR_CPUS

•   int "Maximum number of CPUs (2-32)"
  

•   range 2 32
  

•   depends on SMP
  

•   default "8"
  

no quotation mark?

•   help
  

•     On multiprocessor machines, U-Boot sets up a stack for each CPU.
  

•     Stack memory is pre-allocated. U-Boot must therefore know the
  

•     maximum number of CPUs that may be present.
  

endmenu diff --git a/arch/riscv/include/asm/global_data.h b/arch/riscv/include/asm/global_data.h index a3a342c6e1..23a5f35af5 100644 --- a/arch/riscv/include/asm/global_data.h +++ b/arch/riscv/include/asm/global_data.h @@ -10,12 +10,17 @@ #ifndef __ASM_GBL_DATA_H #define __ASM_GBL_DATA_H

+#include <asm/smp.h>

/* Architecture-specific global data */ struct arch_global_data { long boot_hart; /* boot hart id */ #ifdef CONFIG_SIFIVE_CLINT void __iomem *clint; /* clint base address */ #endif +#ifdef CONFIG_SMP

•   struct ipi_data ipi[CONFIG_NR_CPUS];
  

+#endif };

#include <asm-generic/global_data.h> diff --git a/arch/riscv/include/asm/smp.h b/arch/riscv/include/asm/smp.h new file mode 100644 index 0000000000..bc863fdbaf --- /dev/null +++ b/arch/riscv/include/asm/smp.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/*

• • Copyright (C) 2019 Fraunhofer AISEC,
• • Lukas Auer lukas.auer@aisec.fraunhofer.de
• */

+#ifndef _ASM_RISCV_SMP_H +#define _ASM_RISCV_SMP_H

+/**

• • struct ipi_data - Inter-processor interrupt (IPI) data structure
• • IPIs are used for SMP support to communicate to other harts what function to
• • call. Functions are in the form
• • void (*addr)(ulong hart, ulong arg0, ulong arg1).
• • The function address and the two arguments, arg0 and arg1, are stored in the
• • IPI data structure. The hart ID is inserted by the hart handling the IPI and
• • calling the function.
• • @addr: Address of function
• • @arg0: First argument of function
• • @arg1: Second argument of function
• */

+struct ipi_data {

•   ulong addr;
  

•   ulong arg0;
  

•   ulong arg1;
  

+};

+/**

• • handle_ipi() - interrupt handler for software interrupts
• • The IPI interrupt handler must be called to handle software interrupts. It
• • calls the function specified in the hart's IPI data structure.
• • @hart: Hart ID of the current hart
• */

+void handle_ipi(ulong hart);

+/**

• • smp_call_function() - Call a function on all other harts
• • Send IPIs with the specified function call to all harts.
• • @addr: Address of function
• • @arg0: First argument of function
• • @arg1: Second argument of function
• • @return 0 if OK, -ve on error
• */

+int smp_call_function(ulong addr, ulong arg0, ulong arg1);

+#endif diff --git a/arch/riscv/lib/Makefile b/arch/riscv/lib/Makefile index edfa61690c..19370f9749 100644 --- a/arch/riscv/lib/Makefile +++ b/arch/riscv/lib/Makefile @@ -14,6 +14,7 @@ obj-$(CONFIG_SIFIVE_CLINT) += sifive_clint.o obj-y += interrupts.o obj-y += reset.o obj-y += setjmp.o +obj-$(CONFIG_SMP) += smp.o

# For building EFI apps CFLAGS_$(EFI_CRT0) := $(CFLAGS_EFI) diff --git a/arch/riscv/lib/smp.c b/arch/riscv/lib/smp.c new file mode 100644 index 0000000000..1266a2a0ef --- /dev/null +++ b/arch/riscv/lib/smp.c @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: GPL-2.0+ +/*

• • Copyright (C) 2019 Fraunhofer AISEC,
• • Lukas Auer lukas.auer@aisec.fraunhofer.de
• */

+#include <common.h> +#include <dm.h> +#include <asm/barrier.h> +#include <asm/smp.h>

+DECLARE_GLOBAL_DATA_PTR;

+/**

• • riscv_send_ipi() - Send inter-processor interrupt (IPI)
• • Platform code must provide this function.
• • @hart: Hart ID of receiving hart
• • @return 0 if OK, -ve on error
• */

+extern int riscv_send_ipi(int hart);

+/**

• • riscv_clear_ipi() - Clear inter-processor interrupt (IPI)
• • Platform code must provide this function.
• • @hart: Hart ID of hart to be cleared
• • @return 0 if OK, -ve on error
• */

+extern int riscv_clear_ipi(int hart);

+static int send_ipi_many(struct ipi_data *ipi) +{

•   ofnode node, cpus;
  

•   u32 reg;
  

•   int ret;
  

•   cpus = ofnode_path("/cpus");
  

•   if (!ofnode_valid(cpus)) {
  

•           pr_err("Can't find cpus node!\n");
  

•           return -EINVAL;
  

•   }
  

•   ofnode_for_each_subnode(node, cpus) {
  

•           if (!ofnode_is_available(node))
  

•                   continue;
  

•           /* read hart ID of CPU */
  

•           ret = ofnode_read_u32(node, "reg", &reg);
  

•           if (ret)
  

•                   continue;
  

•           /* skip if it is the hart we are running on */
  

•           if (reg == gd->arch.boot_hart)
  

•                   continue;
  

•           if (reg >= CONFIG_NR_CPUS) {
  

•                   pr_err("Hart ID %d is out of range, increase CONFIG_NR_CPUS\n",
  

•                          reg);
  

•                   continue;
  

•           }
  

•           gd->arch.ipi[reg].addr = ipi->addr;
  

•           gd->arch.ipi[reg].arg0 = ipi->arg0;
  

•           gd->arch.ipi[reg].arg1 = ipi->arg1;
  

•           mb();
  

•           ret = riscv_send_ipi(reg);
  

•           if (ret)
  

•                   return ret;
  

•   }
  

•   return 0;
  

+}

+void handle_ipi(ulong hart) +{

•   int ret;
  

•   void (*smp_function)(ulong hart, ulong arg0, ulong arg1);
  

•   if (hart >= CONFIG_NR_CPUS)
  

•           return;
  

•   ret = riscv_clear_ipi(hart);
  

•   if (ret) {
  

•           pr_err("Cannot clear IPI\n");
  

•           return;
  

•   }
  

•   smp_function = (void (*)(ulong, ulong, ulong))gd->arch.ipi[hart].addr;
  

•   invalidate_icache_all();
  

•   smp_function(hart, gd->arch.ipi[hart].arg0, gd->arch.ipi[hart].arg1);
  

+}

+int smp_call_function(ulong addr, ulong arg0, ulong arg1) +{

•   int ret = 0;
  

•   struct ipi_data ipi;
  

•   ipi.addr = addr;
  

•   ipi.arg0 = arg0;
  

•   ipi.arg1 = arg1;
  

•   ret = send_ipi_many(&ipi);
  

•   return ret;
  

+}

Regards, Bin

On Tue, Feb 12, 2019 at 3:44 AM Lukas Auer lukas.auer@aisec.fraunhofer.de wrote:

Harts on RISC-V boot independently and U-Boot is responsible for managing them. Functions are called on other harts with smp_call_function(), which sends inter-processor interrupts (IPIs) to all other harts. Functions are specified with their address and two function arguments (argument 2 and 3). The first function argument is always the hart ID of the hart calling the function. On the other harts, the IPI interrupt handler handle_ipi() must be called on software interrupts to handle the request and call the specified function.

Functions are stored in the ipi_data data structure. Every hart has its own data structure in global data. While this is not required at the moment (all harts are expected to boot Linux), this does allow future expansion, where other harts may be used for monitoring or other tasks.

Signed-off-by: Lukas Auer lukas.auer@aisec.fraunhofer.de

arch/riscv/Kconfig | 19 +++++ arch/riscv/include/asm/global_data.h | 5 ++ arch/riscv/include/asm/smp.h | 53 +++++++++++++ arch/riscv/lib/Makefile | 1 + arch/riscv/lib/smp.c | 110 +++++++++++++++++++++++++++ 5 files changed, 188 insertions(+) create mode 100644 arch/riscv/include/asm/smp.h create mode 100644 arch/riscv/lib/smp.c

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index c45e4d73a8..c0842178dd 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -116,4 +116,23 @@ config RISCV_RDTIME config SYS_MALLOC_F_LEN default 0x1000

+config SMP

•   bool "Symmetric Multi-Processing"
  

•   help
  

•     This enables support for systems with more than one CPU. If
  

•     you say N here, U-Boot will run on single and multiprocessor
  

•     machines, but will use only one CPU of a multiprocessor
  

•     machine. If you say Y here, U-Boot will run on many, but not
  

•     all, single processor machines.
  

+config NR_CPUS

•   int "Maximum number of CPUs (2-32)"
  

•   range 2 32
  

•   depends on SMP
  

•   default "8"
  

•   help
  

•     On multiprocessor machines, U-Boot sets up a stack for each CPU.
  

•     Stack memory is pre-allocated. U-Boot must therefore know the
  

•     maximum number of CPUs that may be present.
  

endmenu diff --git a/arch/riscv/include/asm/global_data.h b/arch/riscv/include/asm/global_data.h index a3a342c6e1..23a5f35af5 100644 --- a/arch/riscv/include/asm/global_data.h +++ b/arch/riscv/include/asm/global_data.h @@ -10,12 +10,17 @@ #ifndef __ASM_GBL_DATA_H #define __ASM_GBL_DATA_H

+#include <asm/smp.h>

/* Architecture-specific global data */ struct arch_global_data { long boot_hart; /* boot hart id */ #ifdef CONFIG_SIFIVE_CLINT void __iomem *clint; /* clint base address */ #endif +#ifdef CONFIG_SMP

•   struct ipi_data ipi[CONFIG_NR_CPUS];
  

The data passed by "main HART" via global data to other HARTs is not visible reliably and randomly few HARTs fail to enter Linux kernel on my board. I am suspecting per-HART "ipi" data in global data not being cache-line aligned as the cause of behavior but there could be other issue too.

I have a hack which works reliable for me. As-per this hack, we add "mdelay(10)" just before calling riscv_send_ipi() in send_ipi_many(). This hack helped me conclude that there is some sync issue in per-HART "ipi" data.

The above issue is not seen on QEMU so we are fine there.

I would suggest to make per-HART "ipi" data cache-line aligned (just like Linux kernel).

Regards, Anup

On Mon, 2019-02-18 at 10:01 +0000, Auer, Lukas wrote:

On Mon, 2019-02-18 at 10:28 +0530, Anup Patel wrote:

...

On Tue, Feb 12, 2019 at 3:44 AM Lukas Auer lukas.auer@aisec.fraunhofer.de wrote:

...

Harts on RISC-V boot independently and U-Boot is responsible for managing them. Functions are called on other harts with smp_call_function(), which sends inter-processor interrupts (IPIs) to all other harts. Functions are specified with their address and two function arguments (argument 2 and 3). The first function argument is always the hart ID of the hart calling the function. On the other harts, the IPI interrupt handler handle_ipi() must be called on software interrupts to handle the request and call the specified function.

Functions are stored in the ipi_data data structure. Every hart has its own data structure in global data. While this is not required at the moment (all harts are expected to boot Linux), this does allow future expansion, where other harts may be used for monitoring or other tasks.

Signed-off-by: Lukas Auer lukas.auer@aisec.fraunhofer.de

arch/riscv/Kconfig | 19 +++++ arch/riscv/include/asm/global_data.h | 5 ++ arch/riscv/include/asm/smp.h | 53 +++++++++++++ arch/riscv/lib/Makefile | 1 + arch/riscv/lib/smp.c | 110 +++++++++++++++++++++++++++ 5 files changed, 188 insertions(+) create mode 100644 arch/riscv/include/asm/smp.h create mode 100644 arch/riscv/lib/smp.c

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index c45e4d73a8..c0842178dd 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -116,4 +116,23 @@ config RISCV_RDTIME config SYS_MALLOC_F_LEN default 0x1000

+config SMP

•   bool "Symmetric Multi-Processing"
  

•   help
  

•     This enables support for systems with more than one
  

CPU. If

•     you say N here, U-Boot will run on single and
  

multiprocessor

•     machines, but will use only one CPU of a multiprocessor
  

•     machine. If you say Y here, U-Boot will run on many,
  

but not

•     all, single processor machines.
  

+config NR_CPUS

•   int "Maximum number of CPUs (2-32)"
  

•   range 2 32
  

•   depends on SMP
  

•   default "8"
  

•   help
  

•     On multiprocessor machines, U-Boot sets up a stack for
  

each CPU.

•     Stack memory is pre-allocated. U-Boot must therefore
  

know the

•     maximum number of CPUs that may be present.
  

endmenu diff --git a/arch/riscv/include/asm/global_data.h b/arch/riscv/include/asm/global_data.h index a3a342c6e1..23a5f35af5 100644 --- a/arch/riscv/include/asm/global_data.h +++ b/arch/riscv/include/asm/global_data.h @@ -10,12 +10,17 @@ #ifndef __ASM_GBL_DATA_H #define __ASM_GBL_DATA_H

+#include <asm/smp.h>

/* Architecture-specific global data */ struct arch_global_data { long boot_hart; /* boot hart id */ #ifdef CONFIG_SIFIVE_CLINT void __iomem *clint; /* clint base address */ #endif +#ifdef CONFIG_SMP

•   struct ipi_data ipi[CONFIG_NR_CPUS];
  

The data passed by "main HART" via global data to other HARTs is not visible reliably and randomly few HARTs fail to enter Linux kernel on my board. I am suspecting per-HART "ipi" data in global data not being cache-line aligned as the cause of behavior but there could be other issue too.

I have a hack which works reliable for me. As-per this hack, we add "mdelay(10)" just before calling riscv_send_ipi() in send_ipi_many(). This hack helped me conclude that there is some sync issue in per- HART "ipi" data.

The above issue is not seen on QEMU so we are fine there.

I would suggest to make per-HART "ipi" data cache-line aligned (just like Linux kernel).

Interesting, this is definitely a memory coherency issue, probably inadequate fences. I am not sure if making the ipi data structure cache-line aligned will reliably fix it. I'll try to replicate the issue and get it fixed. Thanks for reporting it!

Not sure if it is connected, but I have noticed a regression with the current OpenSBI. Testing U-Boot with the SMP patches applied on QEMU with 4 harts, hart 4 will not receive software interrupts. I have bisected the issue to the following commit.

918c1354b75c74b62f67c4e929551d643f035443 is the first bad commit commit 918c1354b75c74b62f67c4e929551d643f035443 Author: Nick Kossifidis mickflemm@gmail.com Date: Sun Feb 17 09:00:20 2019 +0200

lib: Improve delivery of SBI_IPI_EVENT_HALT

When sbi_ipi_send_many gets called with the current hartid included on pmask (or when pmask is NULL), and we send a HALT event, since the for loop works sequentially over all hartids on the mask, we may send a HALT event to the current hart before the loop finishes. So we will halt the current hart before it can deliver a HALT IPI to the rest and some harts will remain active.

Make sure we send an IPI to the current hart after we've finished with everybody else.

Signed-off-by: Nick Kossifidis mick@ics.forth.gr

I don't have time to look into it right now, but I will do so later.

Thanks, Lukas

On Mon, 2019-02-18 at 18:46 +0530, Anup Patel wrote:

On Mon, Feb 18, 2019 at 5:11 PM Auer, Lukas lukas.auer@aisec.fraunhofer.de wrote:

...

On Mon, 2019-02-18 at 10:01 +0000, Auer, Lukas wrote:

...

On Mon, 2019-02-18 at 10:28 +0530, Anup Patel wrote:

...

On Tue, Feb 12, 2019 at 3:44 AM Lukas Auer lukas.auer@aisec.fraunhofer.de wrote:

...

Harts on RISC-V boot independently and U-Boot is responsible for managing them. Functions are called on other harts with smp_call_function(), which sends inter-processor interrupts (IPIs) to all other harts. Functions are specified with their address and two function arguments (argument 2 and 3). The first function argument is always the hart ID of the hart calling the function. On the other harts, the IPI interrupt handler handle_ipi() must be called on software interrupts to handle the request and call the specified function.

Functions are stored in the ipi_data data structure. Every hart has its own data structure in global data. While this is not required at the moment (all harts are expected to boot Linux), this does allow future expansion, where other harts may be used for monitoring or other tasks.

Signed-off-by: Lukas Auer lukas.auer@aisec.fraunhofer.de

arch/riscv/Kconfig | 19 +++++ arch/riscv/include/asm/global_data.h | 5 ++ arch/riscv/include/asm/smp.h | 53 +++++++++++++ arch/riscv/lib/Makefile | 1 + arch/riscv/lib/smp.c | 110 +++++++++++++++++++++++++++ 5 files changed, 188 insertions(+) create mode 100644 arch/riscv/include/asm/smp.h create mode 100644 arch/riscv/lib/smp.c

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index c45e4d73a8..c0842178dd 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -116,4 +116,23 @@ config RISCV_RDTIME config SYS_MALLOC_F_LEN default 0x1000

+config SMP

•   bool "Symmetric Multi-Processing"
  

•   help
  

•     This enables support for systems with more than one
  

CPU. If

•     you say N here, U-Boot will run on single and
  

multiprocessor

•     machines, but will use only one CPU of a
  

multiprocessor

•     machine. If you say Y here, U-Boot will run on
  

many, but not

•     all, single processor machines.
  

+config NR_CPUS

•   int "Maximum number of CPUs (2-32)"
  

•   range 2 32
  

•   depends on SMP
  

•   default "8"
  

•   help
  

•     On multiprocessor machines, U-Boot sets up a stack
  

for each CPU.

•     Stack memory is pre-allocated. U-Boot must
  

therefore know the

•     maximum number of CPUs that may be present.
  

endmenu diff --git a/arch/riscv/include/asm/global_data.h b/arch/riscv/include/asm/global_data.h index a3a342c6e1..23a5f35af5 100644 --- a/arch/riscv/include/asm/global_data.h +++ b/arch/riscv/include/asm/global_data.h @@ -10,12 +10,17 @@ #ifndef __ASM_GBL_DATA_H #define __ASM_GBL_DATA_H

+#include <asm/smp.h>

/* Architecture-specific global data */ struct arch_global_data { long boot_hart; /* boot hart id */ #ifdef CONFIG_SIFIVE_CLINT void __iomem *clint; /* clint base address */ #endif +#ifdef CONFIG_SMP

•   struct ipi_data ipi[CONFIG_NR_CPUS];
  

The data passed by "main HART" via global data to other HARTs is not visible reliably and randomly few HARTs fail to enter Linux kernel on my board. I am suspecting per-HART "ipi" data in global data not being cache-line aligned as the cause of behavior but there could be other issue too.

I have a hack which works reliable for me. As-per this hack, we add "mdelay(10)" just before calling riscv_send_ipi() in send_ipi_many(). This hack helped me conclude that there is some sync issue in per- HART "ipi" data.

The above issue is not seen on QEMU so we are fine there.

I would suggest to make per-HART "ipi" data cache-line aligned (just like Linux kernel).

Interesting, this is definitely a memory coherency issue, probably inadequate fences. I am not sure if making the ipi data structure cache-line aligned will reliably fix it. I'll try to replicate the issue and get it fixed. Thanks for reporting it!

Not sure if it is connected, but I have noticed a regression with the current OpenSBI. Testing U-Boot with the SMP patches applied on QEMU with 4 harts, hart 4 will not receive software interrupts. I have bisected the issue to the following commit.

918c1354b75c74b62f67c4e929551d643f035443 is the first bad commit commit 918c1354b75c74b62f67c4e929551d643f035443 Author: Nick Kossifidis mickflemm@gmail.com Date: Sun Feb 17 09:00:20 2019 +0200

lib: Improve delivery of SBI_IPI_EVENT_HALT

When sbi_ipi_send_many gets called with the current hartid
included on pmask (or when pmask is NULL), and we send
a HALT event, since the for loop works sequentially over
all hartids on the mask, we may send a HALT event to the
current hart before the loop finishes. So we will halt
the current hart before it can deliver a HALT IPI to the
rest and some harts will remain active.

Make sure we send an IPI to the current hart after we've
finished with everybody else.

Signed-off-by: Nick Kossifidis <mick@ics.forth.gr>

No issue. Already fixed it with following commit:

lib: Fix mask shift in sbi_ipi_send_many()

The mask shift in for-loop of sbi_ipi_send_many() is broken with commit 918c135 ("lib: Improve delivery of SBI_IPI_EVENT_HALT")

This patch fix it.

Signed-off-by: Anup Patel anup.patel@wdc.com

Please pull latest OpenSBI repo.

Thanks, that fixed it.

Lukas

On Tue, 2019-02-19 at 13:46 +0530, Anup Patel wrote:

On Mon, Feb 18, 2019 at 5:11 PM Auer, Lukas lukas.auer@aisec.fraunhofer.de wrote:

...

On Mon, 2019-02-18 at 10:01 +0000, Auer, Lukas wrote:

...

On Mon, 2019-02-18 at 10:28 +0530, Anup Patel wrote:

...

On Tue, Feb 12, 2019 at 3:44 AM Lukas Auer lukas.auer@aisec.fraunhofer.de wrote:

...

Harts on RISC-V boot independently and U-Boot is responsible for managing them. Functions are called on other harts with smp_call_function(), which sends inter-processor interrupts (IPIs) to all other harts. Functions are specified with their address and two function arguments (argument 2 and 3). The first function argument is always the hart ID of the hart calling the function. On the other harts, the IPI interrupt handler handle_ipi() must be called on software interrupts to handle the request and call the specified function.

Functions are stored in the ipi_data data structure. Every hart has its own data structure in global data. While this is not required at the moment (all harts are expected to boot Linux), this does allow future expansion, where other harts may be used for monitoring or other tasks.

Signed-off-by: Lukas Auer lukas.auer@aisec.fraunhofer.de

arch/riscv/Kconfig | 19 +++++ arch/riscv/include/asm/global_data.h | 5 ++ arch/riscv/include/asm/smp.h | 53 +++++++++++++ arch/riscv/lib/Makefile | 1 + arch/riscv/lib/smp.c | 110 +++++++++++++++++++++++++++ 5 files changed, 188 insertions(+) create mode 100644 arch/riscv/include/asm/smp.h create mode 100644 arch/riscv/lib/smp.c

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index c45e4d73a8..c0842178dd 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -116,4 +116,23 @@ config RISCV_RDTIME config SYS_MALLOC_F_LEN default 0x1000

+config SMP

•   bool "Symmetric Multi-Processing"
  

•   help
  

•     This enables support for systems with more than one
  

CPU. If

•     you say N here, U-Boot will run on single and
  

multiprocessor

•     machines, but will use only one CPU of a
  

multiprocessor

•     machine. If you say Y here, U-Boot will run on
  

many, but not

•     all, single processor machines.
  

+config NR_CPUS

•   int "Maximum number of CPUs (2-32)"
  

•   range 2 32
  

•   depends on SMP
  

•   default "8"
  

•   help
  

•     On multiprocessor machines, U-Boot sets up a stack
  

for each CPU.

•     Stack memory is pre-allocated. U-Boot must
  

therefore know the

•     maximum number of CPUs that may be present.
  

endmenu diff --git a/arch/riscv/include/asm/global_data.h b/arch/riscv/include/asm/global_data.h index a3a342c6e1..23a5f35af5 100644 --- a/arch/riscv/include/asm/global_data.h +++ b/arch/riscv/include/asm/global_data.h @@ -10,12 +10,17 @@ #ifndef __ASM_GBL_DATA_H #define __ASM_GBL_DATA_H

+#include <asm/smp.h>

/* Architecture-specific global data */ struct arch_global_data { long boot_hart; /* boot hart id */ #ifdef CONFIG_SIFIVE_CLINT void __iomem *clint; /* clint base address */ #endif +#ifdef CONFIG_SMP

•   struct ipi_data ipi[CONFIG_NR_CPUS];
  

The data passed by "main HART" via global data to other HARTs is not visible reliably and randomly few HARTs fail to enter Linux kernel on my board. I am suspecting per-HART "ipi" data in global data not being cache-line aligned as the cause of behavior but there could be other issue too.

I have a hack which works reliable for me. As-per this hack, we add "mdelay(10)" just before calling riscv_send_ipi() in send_ipi_many(). This hack helped me conclude that there is some sync issue in per- HART "ipi" data.

The above issue is not seen on QEMU so we are fine there.

I would suggest to make per-HART "ipi" data cache-line aligned (just like Linux kernel).

Interesting, this is definitely a memory coherency issue, probably inadequate fences. I am not sure if making the ipi data structure cache-line aligned will reliably fix it. I'll try to replicate the issue and get it fixed. Thanks for reporting it!

Not sure if it is connected, but I have noticed a regression with the current OpenSBI. Testing U-Boot with the SMP patches applied on QEMU with 4 harts, hart 4 will not receive software interrupts. I have bisected the issue to the following commit.

It's not related to OpenSBI IPI issue we fixed recently. I was seeing this issue before OpenSBI IPI breakage.

I tried again with latest OpenSBI and it worked for me 8 out-of 10 times.

Strangely, Atish and you don't see this issue. Maybe it's just my board.

In any case, it will be good to have:

1. Few more fences (taking reference from Linux code)
2. Ensure per-HART "ipi" data is cache line aligned

Maybe with above things in-place, it will become unlikely for me to see this issue.

You can even address this issue as separate patches later since it's only me seeing it occasionally.

I was unable to reproduce this issue. I did however see strange behavior when using the reset button to reset the board [1]. Did you use the reset button to reset the board?

In any case, I will move the memory barrier to the handle_ipi() function. It should not be needed in send_ipi_many, since the IO access function already include suitable memory barriers. We do not have those on the handle_ipi side, so this could actually be the cause of the issue you are seeing.

Thanks, Lukas

[1]: https://github.com/riscv/opensbi/issues/65

-----Original Message----- From: Lukas Auer [mailto:lukas.auer@aisec.fraunhofer.de] Sent: Tuesday, February 12, 2019 3:44 AM To: u-boot@lists.denx.de Cc: Atish Patra Atish.Patra@wdc.com; Anup Patel Anup.Patel@wdc.com; Bin Meng bmeng.cn@gmail.com; Andreas Schwab schwab@suse.de; Palmer Dabbelt palmer@sifive.com; Alexander Graf agraf@suse.de; Lukas Auer lukas.auer@aisec.fraunhofer.de; Rick Chen rick@andestech.com Subject: [PATCH 2/7] riscv: import the supervisor binary interface header file

Import the supervisor binary interface (SBI) header file from Linux (arch/riscv/include/asm/sbi.h). The last change to it was in commit 6d60b6ee0c97 ("RISC-V: Device, timer, IRQs, and the SBI").

Signed-off-by: Lukas Auer lukas.auer@aisec.fraunhofer.de

arch/riscv/include/asm/sbi.h | 94 ++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 arch/riscv/include/asm/sbi.h

diff --git a/arch/riscv/include/asm/sbi.h b/arch/riscv/include/asm/sbi.h new file mode 100644 index 0000000000..ced57defdd --- /dev/null +++ b/arch/riscv/include/asm/sbi.h @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/*

• • Copyright (C) 2015 Regents of the University of California
• • Taken from Linux arch/riscv/include/asm/sbi.h */

+#ifndef _ASM_RISCV_SBI_H +#define _ASM_RISCV_SBI_H

+#include <linux/types.h>

+#define SBI_SET_TIMER 0 +#define SBI_CONSOLE_PUTCHAR 1 +#define SBI_CONSOLE_GETCHAR 2 +#define SBI_CLEAR_IPI 3 +#define SBI_SEND_IPI 4 +#define SBI_REMOTE_FENCE_I 5 +#define SBI_REMOTE_SFENCE_VMA 6 +#define SBI_REMOTE_SFENCE_VMA_ASID 7 +#define SBI_SHUTDOWN 8

+#define SBI_CALL(which, arg0, arg1, arg2) ({ \

• register uintptr_t a0 asm ("a0") = (uintptr_t)(arg0); \
• register uintptr_t a1 asm ("a1") = (uintptr_t)(arg1); \
• register uintptr_t a2 asm ("a2") = (uintptr_t)(arg2); \
• register uintptr_t a7 asm ("a7") = (uintptr_t)(which); \
• asm volatile ("ecall" \

•       : "+r" (a0)				\
  

•       : "r" (a1), "r" (a2), "r" (a7)		\
  

•       : "memory");				\
  

• a0; \

+})

+/* Lazy implementations until SBI is finalized */ #define +SBI_CALL_0(which) SBI_CALL(which, 0, 0, 0) #define SBI_CALL_1(which, +arg0) SBI_CALL(which, arg0, 0, 0) #define SBI_CALL_2(which, arg0, arg1) +SBI_CALL(which, arg0, arg1, 0)

+static inline void sbi_console_putchar(int ch) {

• SBI_CALL_1(SBI_CONSOLE_PUTCHAR, ch);

+}

+static inline int sbi_console_getchar(void) {

• return SBI_CALL_0(SBI_CONSOLE_GETCHAR); }

+static inline void sbi_set_timer(uint64_t stime_value) { #if +__riscv_xlen == 32

• SBI_CALL_2(SBI_SET_TIMER, stime_value, stime_value >> 32); #else
• SBI_CALL_1(SBI_SET_TIMER, stime_value); #endif }

+static inline void sbi_shutdown(void) +{

• SBI_CALL_0(SBI_SHUTDOWN);

+}

+static inline void sbi_clear_ipi(void) +{

• SBI_CALL_0(SBI_CLEAR_IPI);

+}

+static inline void sbi_send_ipi(const unsigned long *hart_mask) {

• SBI_CALL_1(SBI_SEND_IPI, hart_mask);

+}

+static inline void sbi_remote_fence_i(const unsigned long *hart_mask) {

• SBI_CALL_1(SBI_REMOTE_FENCE_I, hart_mask); }

+static inline void sbi_remote_sfence_vma(const unsigned long *hart_mask,

• 			 unsigned long start,
  

• 			 unsigned long size)
  

+{

• SBI_CALL_1(SBI_REMOTE_SFENCE_VMA, hart_mask); }

+static inline void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask,

• 			      unsigned long start,
  

• 			      unsigned long size,
  

• 			      unsigned long asid)
  

+{

• SBI_CALL_1(SBI_REMOTE_SFENCE_VMA_ASID, hart_mask); }

+#endif

2.20.1

Looks good to me.

Reviewed-by: Anup Patel anup.patel@wdc.com

Regards, Anup

-----Original Message----- From: Lukas Auer [mailto:lukas.auer@aisec.fraunhofer.de] Sent: Tuesday, February 12, 2019 3:44 AM To: u-boot@lists.denx.de Cc: Atish Patra Atish.Patra@wdc.com; Anup Patel Anup.Patel@wdc.com; Bin Meng bmeng.cn@gmail.com; Andreas Schwab schwab@suse.de; Palmer Dabbelt palmer@sifive.com; Alexander Graf agraf@suse.de; Lukas Auer lukas.auer@aisec.fraunhofer.de; Rick Chen rick@andestech.com; Anup Patel anup@brainfault.org Subject: [PATCH 4/7] riscv: delay initialization of caches and debug UART

Move the initialization of the caches and the debug UART until after board_init_f_init_reserve. This is in preparation for SMP support, where code prior to this point will be executed by all harts. This ensures that initialization will only be performed once for the main hart running U-Boot.

Signed-off-by: Lukas Auer lukas.auer@aisec.fraunhofer.de

arch/riscv/cpu/start.S | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/arch/riscv/cpu/start.S b/arch/riscv/cpu/start.S index 81ea52b170..a30f6f7194 100644 --- a/arch/riscv/cpu/start.S +++ b/arch/riscv/cpu/start.S @@ -45,10 +45,6 @@ _start: /* mask all interrupts */ csrw MODE_PREFIX(ie), zero

• /* Enable cache */
• jal icache_enable
• jal dcache_enable

/*

• Set stackpointer in internal/ex RAM to call board_init_f

*/ @@ -57,10 +53,6 @@ call_board_init_f: li t1, CONFIG_SYS_INIT_SP_ADDR and sp, t1, t0 /* force 16 byte alignment */

-#ifdef CONFIG_DEBUG_UART

• jal debug_uart_init

-#endif

call_board_init_f_0: mv a0, sp jal board_init_f_alloc_reserve @@ -74,6 +66,14 @@ call_board_init_f_0: /* save the boot hart id to global_data */ SREG s0, GD_BOOT_HART(gp)

• /* Enable cache */
• jal icache_enable
• jal dcache_enable

+#ifdef CONFIG_DEBUG_UART

• jal debug_uart_init

+#endif

• mv a0, zero /* a0 <-- boot_flags = 0 */ la t5, board_init_f jr t5 /* jump to board_init_f() */

-- 2.20.1

Looks good to me.

Reviewed-by: Anup Patel anup.patel@wdc.com

Regards, Anup

On Tue, 2019-02-12 at 01:48 +0000, Anup Patel wrote:

...

-----Original Message----- From: Lukas Auer [mailto:lukas.auer@aisec.fraunhofer.de] Sent: Tuesday, February 12, 2019 3:44 AM To: u-boot@lists.denx.de Cc: Atish Patra Atish.Patra@wdc.com; Anup Patel Anup.Patel@wdc.com; Bin Meng bmeng.cn@gmail.com; Andreas Schwab schwab@suse.de; Palmer Dabbelt palmer@sifive.com; Alexander Graf agraf@suse.de; Lukas Auer lukas.auer@aisec.fraunhofer.de; Anup Patel anup@brainfault.org; Rick Chen rick@andestech.com; Baruch Siach baruch@tkos.co.il; Stefan Roese sr@denx.de Subject: [PATCH 5/7] riscv: add support for multi-hart systems

On RISC-V, all harts boot independently. To be able to run on a multi-hart system, U-Boot must be extended with the functionality to manage all harts in the system. A new config option, CONFIG_MAIN_HART, is used to select the hart U-Boot runs on. All other harts are halted. U-Boot can delegate functions to them using smp_call_function().

Every hart has a valid pointer to the global data structure and a 8KiB stack by default. The stack size is set with CONFIG_STACK_SIZE_SHIFT.

Signed-off-by: Lukas Auer lukas.auer@aisec.fraunhofer.de

arch/riscv/Kconfig | 12 +++++ arch/riscv/cpu/start.S | 102 ++++++++++++++++++++++++++++++++++- arch/riscv/include/asm/csr.h | 1 + 3 files changed, 114 insertions(+), 1 deletion(-)

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 3a51339c4d..af8d0f8d67 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -140,4 +140,16 @@ config SBI_IPI default y if RISCV_SMODE depends on SMP

+config MAIN_HART

• int "Main hart in system"
• default 0
• help

•  Some SoCs include harts of various sizes, some of which might
  

not

•  be suitable for running U-Boot. CONFIG_MAIN_HART is used to
  

select

•  the hart U-Boot runs on.
  

This config option can be avoided altogether if we have lottery based system to select "Main HART" in start.S.

With the MAIN_HART config option in-place, every system will have to pick a "Main HART". What if the "Main HART" itself does not come online due to HW failure.

Good point, I did not consider this. I will add a lottery-based system in the next version.

Thanks, Lukas

-----Original Message----- From: Lukas Auer [mailto:lukas.auer@aisec.fraunhofer.de] Sent: Tuesday, February 12, 2019 3:44 AM To: u-boot@lists.denx.de Cc: Atish Patra Atish.Patra@wdc.com; Anup Patel Anup.Patel@wdc.com; Bin Meng bmeng.cn@gmail.com; Andreas Schwab schwab@suse.de; Palmer Dabbelt palmer@sifive.com; Alexander Graf agraf@suse.de; Lukas Auer lukas.auer@aisec.fraunhofer.de; Anup Patel anup@brainfault.org; Rick Chen rick@andestech.com; Simon Glass sjg@chromium.org Subject: [PATCH 6/7] riscv: boot images passed to bootm on all harts

Signed-off-by: Lukas Auer lukas.auer@aisec.fraunhofer.de

arch/riscv/lib/bootm.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/arch/riscv/lib/bootm.c b/arch/riscv/lib/bootm.c index f36b8702ef..efbd3e23e7 100644 --- a/arch/riscv/lib/bootm.c +++ b/arch/riscv/lib/bootm.c @@ -13,6 +13,7 @@ #include <image.h> #include <asm/byteorder.h> #include <asm/csr.h> +#include <asm/smp.h> #include <dm/device.h> #include <dm/root.h> #include <u-boot/zlib.h> @@ -81,6 +82,9 @@ static void boot_jump_linux(bootm_headers_t *images, int flag) { void (*kernel)(ulong hart, void *dtb); int fake = (flag & BOOTM_STATE_OS_FAKE_GO); +#ifdef CONFIG_SMP

• int ret;

+#endif

kernel = (void (*)(ulong, void *))images->ep;

@@ -92,8 +96,15 @@ static void boot_jump_linux(bootm_headers_t *images, int flag) announce_and_cleanup(fake);

if (!fake) {

• if (IMAGE_ENABLE_OF_LIBFDT && images->ft_len)
  

• if (IMAGE_ENABLE_OF_LIBFDT && images->ft_len) { #ifdef
  

CONFIG_SMP

• 	ret = smp_call_function(images->ep,
  

• 				(ulong)images->ft_addr, 0);
  

• 	if (ret)
  

• 		hang();
  

+#endif kernel(gd->arch.boot_hart, images->ft_addr);

• }
  

  }

}

-- 2.20.1

Looks good to me.

Reviewed-by: Anup Patel anup.patel@wdc.com

Regards, Anup