Newer Intel SoCs have different ways of setting up cache-as-ram (CAR). Add support for these along with suitable configuration options.
To make the code cleaner, adjust a few definitions in processor.h so that they can be used from assembler.
Signed-off-by: Simon Glass sjg@chromium.org ---
Changes in v4: - Adjust - Fix up license header - Fix various code-style problems - Use CONFIG_INTEL_CAR_CQOS to control car2.S inclusion - Use car_init_ret to return - Use post_code() calls consistent with car.S
Changes in v3: - Drop dead code - Drop unneeded Kconfig file - Use a macro for is-power-of-two
Changes in v2: None
arch/x86/Kconfig | 16 + arch/x86/cpu/intel_common/Makefile | 8 + arch/x86/cpu/intel_common/car2.S | 448 ++++++++++++++++++++++++ arch/x86/cpu/intel_common/car2_uninit.S | 87 +++++ arch/x86/include/asm/processor.h | 12 +- 5 files changed, 564 insertions(+), 7 deletions(-) create mode 100644 arch/x86/cpu/intel_common/car2.S create mode 100644 arch/x86/cpu/intel_common/car2_uninit.S
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index ea50b2f54d..3dde26ca93 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -879,4 +879,20 @@ config HIGH_TABLE_SIZE Increse it if the default size does not fit the board's needs. This is most likely due to a large ACPI DSDT table is used.
+config INTEL_CAR_CQOS + bool "Support Intel Cache Quality of Service" + help + Cache Quality of Service allows more fine-grained control of cache + usage. As result, it is possible to set up a portion of L2 cache for + CAR and use the remainder for actual caching. + +# +# Each bit in QOS mask controls this many bytes. This is calculated as: +# (CACHE_WAYS / CACHE_BITS_PER_MASK) * CACHE_LINE_SIZE * CACHE_SETS +# +config CACHE_QOS_SIZE_PER_BIT + hex + depends on INTEL_CAR_CQOS + default 0x20000 # 128 KB + endmenu diff --git a/arch/x86/cpu/intel_common/Makefile b/arch/x86/cpu/intel_common/Makefile index dfbc29f047..09212cee04 100644 --- a/arch/x86/cpu/intel_common/Makefile +++ b/arch/x86/cpu/intel_common/Makefile @@ -8,6 +8,14 @@ obj-$(CONFIG_$(SPL_TPL_)X86_32BIT_INIT) += me_status.o obj-$(CONFIG_$(SPL_TPL_)X86_32BIT_INIT) += report_platform.o obj-$(CONFIG_$(SPL_TPL_)X86_32BIT_INIT) += mrc.o endif + +ifdef CONFIG_INTEL_CAR_CQOS +obj-$(CONFIG_TPL_BUILD) += car2.o +ifndef CONFIG_SPL_BUILD +obj-y += car2_uninit.o +endif +endif + obj-y += cpu.o obj-y += fast_spi.o obj-y += lpc.o diff --git a/arch/x86/cpu/intel_common/car2.S b/arch/x86/cpu/intel_common/car2.S new file mode 100644 index 0000000000..086f987477 --- /dev/null +++ b/arch/x86/cpu/intel_common/car2.S @@ -0,0 +1,448 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file was modified from the coreboot version. + * + * Copyright (C) 2015-2016 Intel Corp. + */ + +#include <config.h> +#include <asm/msr-index.h> +#include <asm/mtrr.h> +#include <asm/post.h> +#include <asm/processor.h> +#include <asm/processor-flags.h> + +#define KiB 1024 + +#define IS_POWER_OF_2(x) (!((x) & ((x) - 1))) + +.global car_init +car_init: + post_code(POST_CAR_START) + + /* + * Use the MTRR default type MSR as a proxy for detecting INIT#. + * Reset the system if any known bits are set in that MSR. That is + * an indication of the CPU not being properly reset. + */ +check_for_clean_reset: + mov $MTRR_DEF_TYPE_MSR, %ecx + rdmsr + and $(MTRR_DEF_TYPE_EN | MTRR_DEF_TYPE_FIX_EN), %eax + cmp $0, %eax + jz no_reset + /* perform warm reset */ + movw $IO_PORT_RESET, %dx + movb $(SYS_RST | RST_CPU), %al + outb %al, %dx + +no_reset: + post_code(POST_CAR_SIPI) + + /* Clear/disable fixed MTRRs */ + mov $fixed_mtrr_list_size, %ebx + xor %eax, %eax + xor %edx, %edx + +clear_fixed_mtrr: + add $-2, %ebx + movzwl fixed_mtrr_list(%ebx), %ecx + wrmsr + jnz clear_fixed_mtrr + + post_code(POST_CAR_MTRR) + + /* Figure put how many MTRRs we have, and clear them out */ + mov $MTRR_CAP_MSR, %ecx + rdmsr + movzb %al, %ebx /* Number of variable MTRRs */ + mov $MTRR_PHYS_BASE_MSR(0), %ecx + xor %eax, %eax + xor %edx, %edx + +clear_var_mtrr: + wrmsr + inc %ecx + wrmsr + inc %ecx + dec %ebx + jnz clear_var_mtrr + + post_code(POST_CAR_UNCACHEABLE) + + /* Configure default memory type to uncacheable (UC) */ + mov $MTRR_DEF_TYPE_MSR, %ecx + rdmsr + /* Clear enable bits and set default type to UC */ + and $~(MTRR_DEF_TYPE_MASK | MTRR_DEF_TYPE_EN | \ + MTRR_DEF_TYPE_FIX_EN), %eax + wrmsr + + /* + * Configure MTRR_PHYS_MASK_HIGH for proper addressing above 4GB + * based on the physical address size supported for this processor + * This is based on read from CPUID EAX = 080000008h, EAX bits [7:0] + * + * Examples: + * MTRR_PHYS_MASK_HIGH = 00000000Fh For 36 bit addressing + * MTRR_PHYS_MASK_HIGH = 0000000FFh For 40 bit addressing + */ + + movl $0x80000008, %eax /* Address sizes leaf */ + cpuid + sub $32, %al + movzx %al, %eax + xorl %esi, %esi + bts %eax, %esi + dec %esi /* esi <- MTRR_PHYS_MASK_HIGH */ + + post_code(POST_CAR_BASE_ADDRESS) + +#if IS_POWER_OF_2(CONFIG_DCACHE_RAM_SIZE) + /* Configure CAR region as write-back (WB) */ + mov $MTRR_PHYS_BASE_MSR(0), %ecx + mov $CONFIG_DCACHE_RAM_BASE, %eax + or $MTRR_TYPE_WRBACK, %eax + xor %edx,%edx + wrmsr + + /* Configure the MTRR mask for the size region */ + mov $MTRR_PHYS_MASK(0), %ecx + mov $CONFIG_DCACHE_RAM_SIZE, %eax /* size mask */ + dec %eax + not %eax + or $MTRR_PHYS_MASK_VALID, %eax + movl %esi, %edx /* edx <- MTRR_PHYS_MASK_HIGH */ + wrmsr +#elif (CONFIG_DCACHE_RAM_SIZE == 768 * KiB) /* 768 KiB */ + /* Configure CAR region as write-back (WB) */ + mov $MTRR_PHYS_BASE_MSR(0), %ecx + mov $CONFIG_DCACHE_RAM_BASE, %eax + or $MTRR_TYPE_WRBACK, %eax + xor %edx,%edx + wrmsr + + mov $MTRR_PHYS_MASK_MSR(0), %ecx + mov $(512 * KiB), %eax /* size mask */ + dec %eax + not %eax + or $MTRR_PHYS_MASK_VALID, %eax + movl %esi, %edx /* edx <- MTRR_PHYS_MASK_HIGH */ + wrmsr + + mov $MTRR_PHYS_BASE_MSR(1), %ecx + mov $(CONFIG_DCACHE_RAM_BASE + 512 * KiB), %eax + or $MTRR_TYPE_WRBACK, %eax + xor %edx,%edx + wrmsr + + mov $MTRR_PHYS_MASK_MSR(1), %ecx + mov $(256 * KiB), %eax /* size mask */ + dec %eax + not %eax + or $MTRR_PHYS_MASK_VALID, %eax + movl %esi, %edx /* edx <- MTRR_PHYS_MASK_HIGH */ + wrmsr +#else +#error "DCACHE_RAM_SIZE is not a power of 2 and setup code is missing" +#endif + post_code(POST_CAR_FILL) + + /* Enable variable MTRRs */ + mov $MTRR_DEF_TYPE_MSR, %ecx + rdmsr + or $MTRR_DEF_TYPE_EN, %eax + wrmsr + + /* Enable caching */ + mov %cr0, %eax + and $~(X86_CR0_CD | X86_CR0_NW), %eax + invd + mov %eax, %cr0 + +#if IS_ENABLED(CONFIG_INTEL_CAR_NEM) + jmp car_nem +#elif IS_ENABLED(CONFIG_INTEL_CAR_CQOS) + jmp car_cqos +#elif IS_ENABLED(CONFIG_INTEL_CAR_NEM_ENHANCED) + jmp car_nem_enhanced +#else +#error "No CAR mechanism selected: +#endif + jmp car_init_ret + +fixed_mtrr_list: + .word MTRR_FIX_64K_00000_MSR + .word MTRR_FIX_16K_80000_MSR + .word MTRR_FIX_16K_A0000_MSR + .word MTRR_FIX_4K_C0000_MSR + .word MTRR_FIX_4K_C8000_MSR + .word MTRR_FIX_4K_D0000_MSR + .word MTRR_FIX_4K_D8000_MSR + .word MTRR_FIX_4K_E0000_MSR + .word MTRR_FIX_4K_E8000_MSR + .word MTRR_FIX_4K_F0000_MSR + .word MTRR_FIX_4K_F8000_MSR +fixed_mtrr_list_size = . - fixed_mtrr_list + +#if IS_ENABLED(CONFIG_INTEL_CAR_NEM) +.global car_nem +car_nem: + /* Disable cache eviction (setup stage) */ + mov $MSR_EVICT_CTL, %ecx + rdmsr + or $0x1, %eax + wrmsr + + post_code(0x26) + + /* Clear the cache memory region. This will also fill up the cache */ + movl $CONFIG_DCACHE_RAM_BASE, %edi + movl $CONFIG_DCACHE_RAM_SIZE, %ecx + shr $0x02, %ecx + xor %eax, %eax + cld + rep stosl + + post_code(0x27) + + /* Disable cache eviction (run stage) */ + mov $MSR_EVICT_CTL, %ecx + rdmsr + or $0x2, %eax + wrmsr + + post_code(0x28) + + jmp car_init_ret + +#elif IS_ENABLED(CONFIG_INTEL_CAR_CQOS) +.global car_cqos +car_cqos: + /* + * Create CBM_LEN_MASK based on CBM_LEN + * Get CPUID.(EAX=10H, ECX=2H):EAX.CBM_LEN[bits 4:0] + */ + mov $0x10, %eax + mov $0x2, %ecx + cpuid + and $0x1f, %eax + add $1, %al + + mov $1, %ebx + mov %al, %cl + shl %cl, %ebx + sub $1, %ebx + + /* Store the CBM_LEN_MASK in mm3 for later use */ + movd %ebx, %mm3 + + /* + * Disable both L1 and L2 prefetcher. For yet-to-understood reason, + * prefetchers slow down filling cache with rep stos in CQOS mode. + */ + mov $MSR_PREFETCH_CTL, %ecx + rdmsr + or $(PREFETCH_L1_DISABLE | PREFETCH_L2_DISABLE), %eax + wrmsr + +#if (CONFIG_DCACHE_RAM_SIZE == CONFIG_L2_CACHE_SIZE) +/* + * If CAR size is set to full L2 size, mask is calculated as all-zeros. + * This is not supported by the CPU/uCode. + */ +#error "CQOS CAR may not use whole L2 cache area" +#endif + + /* Calculate how many bits to be used for CAR */ + xor %edx, %edx + mov $CONFIG_DCACHE_RAM_SIZE, %eax /* dividend */ + mov $CONFIG_CACHE_QOS_SIZE_PER_BIT, %ecx /* divisor */ + div %ecx /* result is in eax */ + mov %eax, %ecx /* save to ecx */ + mov $1, %ebx + shl %cl, %ebx + sub $1, %ebx /* resulting mask is is in ebx */ + + /* Set this mask for initial cache fill */ + mov $MSR_L2_QOS_MASK(0), %ecx + rdmsr + mov %ebx, %eax + wrmsr + + /* Set CLOS selector to 0 */ + mov $MSR_IA32_PQR_ASSOC, %ecx + rdmsr + and $~MSR_IA32_PQR_ASSOC_MASK, %edx /* select mask 0 */ + wrmsr + + /* We will need to block CAR region from evicts */ + mov $MSR_L2_QOS_MASK(1), %ecx + rdmsr + /* Invert bits that are to be used for cache */ + mov %ebx, %eax + xor $~0, %eax /* invert 32 bits */ + + /* + * Use CBM_LEN_MASK stored in mm3 to set bits based on Capacity Bit + * Mask Length. + */ + movd %mm3, %ebx + and %ebx, %eax + wrmsr + + post_code(0x26) + + /* Clear the cache memory region. This will also fill up the cache */ + movl $CONFIG_DCACHE_RAM_BASE, %edi + movl $CONFIG_DCACHE_RAM_SIZE, %ecx + shr $0x02, %ecx + xor %eax, %eax + cld + rep stosl + + post_code(0x27) + + /* Cache is populated. Use mask 1 that will block evicts */ + mov $MSR_IA32_PQR_ASSOC, %ecx + rdmsr + and $~MSR_IA32_PQR_ASSOC_MASK, %edx /* clear index bits first */ + or $1, %edx /* select mask 1 */ + wrmsr + + /* Enable prefetchers */ + mov $MSR_PREFETCH_CTL, %ecx + rdmsr + and $~(PREFETCH_L1_DISABLE | PREFETCH_L2_DISABLE), %eax + wrmsr + + post_code(0x28) + + jmp car_init_ret + +#elif IS_ENABLED(CONFIG_INTEL_CAR_NEM_ENHANCED) +.global car_nem_enhanced +car_nem_enhanced: + /* Disable cache eviction (setup stage) */ + mov $MSR_EVICT_CTL, %ecx + rdmsr + or $0x1, %eax + wrmsr + post_code(0x26) + + /* Create n-way set associativity of cache */ + xorl %edi, %edi +find_llc_subleaf: + movl %edi, %ecx + movl $0x04, %eax + cpuid + inc %edi + and $0xe0, %al /* EAX[7:5] = Cache Level */ + cmp $0x60, %al /* Check to see if it is LLC */ + jnz find_llc_subleaf + + /* + * Set MSR 0xC91 IA32_L3_MASK_! = 0xE/0xFE/0xFFE/0xFFFE + * for 4/8/16 way of LLC + */ + shr $22, %ebx + inc %ebx + /* Calculate n-way associativity of LLC */ + mov %bl, %cl + + /* + * Maximizing RO cacheability while locking in the CAR to a + * single way since that particular way won't be victim candidate + * for evictions. + * This has been done after programing LLC_WAY_MASK_1 MSR + * with desired LLC way as mentioned below. + * + * Hence create Code and Data Size as per request + * Code Size (RO) : Up to 16M + * Data Size (RW) : Up to 256K + */ + movl $0x01, %eax + /* + * LLC Ways -> LLC_WAY_MASK_1: + * 4: 0x000E + * 8: 0x00FE + * 12: 0x0FFE + * 16: 0xFFFE + * + * These MSRs contain one bit per each way of LLC + * - If this bit is '0' - the way is protected from eviction + * - If this bit is '1' - the way is not protected from eviction + */ + shl %cl, %eax + subl $0x02, %eax + movl $MSR_IA32_L3_MASK_1, %ecx + xorl %edx, %edx + wrmsr + /* + * Set MSR 0xC92 IA32_L3_MASK_2 = 0x1 + * + * For SKL SOC, data size remains 256K consistently. + * Hence, creating 1-way associative cache for Data + */ + mov $MSR_IA32_L3_MASK_2, %ecx + mov $0x01, %eax + xorl %edx, %edx + wrmsr + /* + * Set MSR_IA32_PQR_ASSOC = 0x02 + * + * Possible values: + * 0: Default value, no way mask should be applied + * 1: Apply way mask 1 to LLC + * 2: Apply way mask 2 to LLC + * 3: Shouldn't be use in NEM Mode + */ + movl $MSR_IA32_PQR_ASSOC, %ecx + movl $0x02, %eax + xorl %edx, %edx + wrmsr + + movl $CONFIG_DCACHE_RAM_BASE, %edi + movl $CONFIG_DCACHE_RAM_SIZE, %ecx + shr $0x02, %ecx + xor %eax, %eax + cld + rep stosl + /* + * Set MSR_IA32_PQR_ASSOC = 0x01 + * At this stage we apply LLC_WAY_MASK_1 to the cache. + * i.e. way 0 is protected from eviction. + */ + movl $MSR_IA32_PQR_ASSOC, %ecx + movl $0x01, %eax + xorl %edx, %edx + wrmsr + + post_code(0x27) + /* + * Enable No-Eviction Mode Run State by setting + * NO_EVICT_MODE MSR 2E0h bit [1] = '1'. + */ + + movl $MSR_EVICT_CTL, %ecx + rdmsr + orl $0x02, %eax + wrmsr + + post_code(0x28) + + jmp car_init_ret +#endif + +#if CONFIG_IS_ENABLED(X86_16BIT_INIT) +_dt_ucode_base_size: + /* These next two fields are filled in by binman */ +.globl ucode_base +ucode_base: /* Declared in microcode.h */ + .long 0 /* microcode base */ +.globl ucode_size +ucode_size: /* Declared in microcode.h */ + .long 0 /* microcode size */ + .long CONFIG_SYS_MONITOR_BASE /* code region base */ + .long CONFIG_SYS_MONITOR_LEN /* code region size */ +#endif diff --git a/arch/x86/cpu/intel_common/car2_uninit.S b/arch/x86/cpu/intel_common/car2_uninit.S new file mode 100644 index 0000000000..aba3a5381e --- /dev/null +++ b/arch/x86/cpu/intel_common/car2_uninit.S @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright 2017 Intel Corp. + * Copyright 2019 Google LLC + * Taken from coreboot file exit_car.S + */ + +#include <config.h> +#include <asm/msr-index.h> +#include <asm/mtrr.h> + +.text +.global car_uninit +car_uninit: + + /* + * Retrieve return address from stack as it will get trashed below if + * execution is utilizing the cache-as-ram stack. + */ + pop %ebx + + /* Disable MTRRs */ + mov $(MTRR_DEF_TYPE_MSR), %ecx + rdmsr + and $(~(MTRR_DEF_TYPE_EN | MTRR_DEF_TYPE_FIX_EN)), %eax + wrmsr + +#ifdef CONFIG_INTEL_CAR_NEM +.global car_nem_teardown +car_nem_teardown: + + /* invalidate cache contents */ + invd + + /* Knock down bit 1 then bit 0 of NEM control not combining steps */ + mov $(MSR_EVICT_CTL), %ecx + rdmsr + and $(~(1 << 1)), %eax + wrmsr + and $(~(1 << 0)), %eax + wrmsr + +#elif IS_ENABLED(CONFIG_INTEL_CAR_CQOS) +.global car_cqos_teardown +car_cqos_teardown: + + /* Go back to all-evicting mode, set both masks to all-1s */ + mov $MSR_L2_QOS_MASK(0), %ecx + rdmsr + mov $~0, %al + wrmsr + + mov $MSR_L2_QOS_MASK(1), %ecx + rdmsr + mov $~0, %al + wrmsr + + /* Reset CLOS selector to 0 */ + mov $MSR_IA32_PQR_ASSOC, %ecx + rdmsr + and $~MSR_IA32_PQR_ASSOC_MASK, %edx + wrmsr + +#elif IS_ENABLED(CONFIG_INTEL_CAR_NEM_ENHANCED) +.global car_nem_enhanced_teardown +car_nem_enhanced_teardown: + + /* invalidate cache contents */ + invd + + /* Knock down bit 1 then bit 0 of NEM control not combining steps */ + mov $(MSR_EVICT_CTL), %ecx + rdmsr + and $(~(1 << 1)), %eax + wrmsr + and $(~(1 << 0)), %eax + wrmsr + + /* Reset CLOS selector to 0 */ + mov $IA32_PQR_ASSOC, %ecx + rdmsr + and $~IA32_PQR_ASSOC_MASK, %edx + wrmsr +#endif + + /* Return to caller */ + jmp *%ebx diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index f1d9977bcb..d7b6836786 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -25,8 +25,6 @@ /* Length of the public header on Intel microcode blobs */ #define UCODE_HEADER_LEN 0x30
-#ifndef __ASSEMBLY__ - /* * This register is documented in (for example) the Intel Atom Processor E3800 * Product Family Datasheet in "PCU - Power Management Controller (PMC)". @@ -37,11 +35,11 @@ */ #define IO_PORT_RESET 0xcf9
-enum { - SYS_RST = 1 << 1, /* 0 for soft reset, 1 for hard reset */ - RST_CPU = 1 << 2, /* initiate reset */ - FULL_RST = 1 << 3, /* full power cycle */ -}; +#define SYS_RST (1 << 1) /* 0 for soft reset, 1 for hard reset */ +#define RST_CPU (1 << 2) /* initiate reset */ +#define FULL_RST (1 << 3) /* full power cycle */ + +#ifndef __ASSEMBLY__
static inline __attribute__((always_inline)) void cpu_hlt(void) {