Align the allocation size with DMA_MINALIGN to make sure we do not flush/invalidate data from following allocations.

Reviewed-by: Neha Malcom Francis n-francis@ti.com Tested-by: Venkatesh Yadav Abbarapu venkatesh.abbarapu@amd.com Signed-off-by: Neil Armstrong neil.armstrong@linaro.org --- drivers/ufs/ufs.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/drivers/ufs/ufs.c b/drivers/ufs/ufs.c index be64bf971f1..e005cc90608 100644 --- a/drivers/ufs/ufs.c +++ b/drivers/ufs/ufs.c @@ -633,7 +633,9 @@ static int ufshcd_memory_alloc(struct ufs_hba *hba) /* Allocate one Transfer Request Descriptor * Should be aligned to 1k boundary. */ - hba->utrdl = memalign(1024, sizeof(struct utp_transfer_req_desc)); + hba->utrdl = memalign(1024, + ALIGN(sizeof(struct utp_transfer_req_desc), + ARCH_DMA_MINALIGN)); if (!hba->utrdl) { dev_err(hba->dev, "Transfer Descriptor memory allocation failed\n"); return -ENOMEM; @@ -642,7 +644,9 @@ static int ufshcd_memory_alloc(struct ufs_hba *hba) /* Allocate one Command Descriptor * Should be aligned to 1k boundary. */ - hba->ucdl = memalign(1024, sizeof(struct utp_transfer_cmd_desc)); + hba->ucdl = memalign(1024, + ALIGN(sizeof(struct utp_transfer_cmd_desc), + ARCH_DMA_MINALIGN)); if (!hba->ucdl) { dev_err(hba->dev, "Command descriptor memory allocation failed\n"); return -ENOMEM;

There is no need to flush and invalidate all data updated by the driver, mainly because on ARM platforms flush also invalidates the cachelines.

Split the function in two and add the appropriate cacheline invalidates after the UFS DMA operation finishes to make sure we read from memory.

Flushing then invalidating cacheline unaligned data causes data corruption issues on Qualcomm platforms, and is largely unnecessary anyway, so let's cleanup the cache operations.

Reviewed-by: Neha Malcom Francis n-francis@ti.com Tested-by: Venkatesh Yadav Abbarapu venkatesh.abbarapu@amd.com Signed-off-by: Neil Armstrong neil.armstrong@linaro.org --- drivers/ufs/ufs.c | 45 ++++++++++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 15 deletions(-)

diff --git a/drivers/ufs/ufs.c b/drivers/ufs/ufs.c index 3d9a7d7ee12..5845fd694d3 100644 --- a/drivers/ufs/ufs.c +++ b/drivers/ufs/ufs.c @@ -696,17 +696,28 @@ static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba) }

/** - * ufshcd_cache_flush_and_invalidate - Flush and invalidate cache + * ufshcd_cache_flush - Flush cache * - * Flush and invalidate cache in aligned address..address+size range. - * The invalidation is in place to avoid stale data in cache. + * Flush cache in aligned address..address+size range. */ -static void ufshcd_cache_flush_and_invalidate(void *addr, unsigned long size) +static void ufshcd_cache_flush(void *addr, unsigned long size) { uintptr_t start_addr = (uintptr_t)addr & ~(ARCH_DMA_MINALIGN - 1); uintptr_t end_addr = ALIGN((uintptr_t)addr + size, ARCH_DMA_MINALIGN);

flush_dcache_range(start_addr, end_addr); +} + +/** + * ufshcd_cache_invalidate - Invalidate cache + * + * Invalidate cache in aligned address..address+size range. + */ +static void ufshcd_cache_invalidate(void *addr, unsigned long size) +{ + uintptr_t start_addr = (uintptr_t)addr & ~(ARCH_DMA_MINALIGN - 1); + uintptr_t end_addr = ALIGN((uintptr_t)addr + size, ARCH_DMA_MINALIGN); + invalidate_dcache_range(start_addr, end_addr); }

@@ -754,7 +765,7 @@ static void ufshcd_prepare_req_desc_hdr(struct ufs_hba *hba,

req_desc->prd_table_length = 0;

- ufshcd_cache_flush_and_invalidate(req_desc, sizeof(*req_desc)); + ufshcd_cache_flush(req_desc, sizeof(*req_desc)); }

static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba, @@ -785,13 +796,13 @@ static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba, /* Copy the Descriptor */ if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC) { memcpy(ucd_req_ptr + 1, query->descriptor, len); - ufshcd_cache_flush_and_invalidate(ucd_req_ptr, 2 * sizeof(*ucd_req_ptr)); + ufshcd_cache_flush(ucd_req_ptr, 2 * sizeof(*ucd_req_ptr)); } else { - ufshcd_cache_flush_and_invalidate(ucd_req_ptr, sizeof(*ucd_req_ptr)); + ufshcd_cache_flush(ucd_req_ptr, sizeof(*ucd_req_ptr)); }

memset(hba->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); - ufshcd_cache_flush_and_invalidate(hba->ucd_rsp_ptr, sizeof(*hba->ucd_rsp_ptr)); + ufshcd_cache_flush(hba->ucd_rsp_ptr, sizeof(*hba->ucd_rsp_ptr)); }

static inline void ufshcd_prepare_utp_nop_upiu(struct ufs_hba *hba) @@ -809,8 +820,8 @@ static inline void ufshcd_prepare_utp_nop_upiu(struct ufs_hba *hba)

memset(hba->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));

- ufshcd_cache_flush_and_invalidate(ucd_req_ptr, sizeof(*ucd_req_ptr)); - ufshcd_cache_flush_and_invalidate(hba->ucd_rsp_ptr, sizeof(*hba->ucd_rsp_ptr)); + ufshcd_cache_flush(ucd_req_ptr, sizeof(*ucd_req_ptr)); + ufshcd_cache_flush(hba->ucd_rsp_ptr, sizeof(*hba->ucd_rsp_ptr)); }

/** @@ -877,6 +888,8 @@ static int ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag) */ static inline int ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr) { + ufshcd_cache_invalidate(ucd_rsp_ptr, sizeof(*ucd_rsp_ptr)); + return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24; }

@@ -888,6 +901,8 @@ static inline int ufshcd_get_tr_ocs(struct ufs_hba *hba) { struct utp_transfer_req_desc *req_desc = hba->utrdl;

+ ufshcd_cache_invalidate(req_desc, sizeof(*req_desc)); + return le32_to_cpu(req_desc->header.dword_2) & MASK_OCS; }

@@ -1437,8 +1452,8 @@ void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufs_hba *hba, memcpy(ucd_req_ptr->sc.cdb, pccb->cmd, cdb_len);

memset(hba->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); - ufshcd_cache_flush_and_invalidate(ucd_req_ptr, sizeof(*ucd_req_ptr)); - ufshcd_cache_flush_and_invalidate(hba->ucd_rsp_ptr, sizeof(*hba->ucd_rsp_ptr)); + ufshcd_cache_flush(ucd_req_ptr, sizeof(*ucd_req_ptr)); + ufshcd_cache_flush(hba->ucd_rsp_ptr, sizeof(*hba->ucd_rsp_ptr)); }

static inline void prepare_prdt_desc(struct ufshcd_sg_entry *entry, @@ -1461,7 +1476,7 @@ static void prepare_prdt_table(struct ufs_hba *hba, struct scsi_cmd *pccb)

if (!datalen) { req_desc->prd_table_length = 0; - ufshcd_cache_flush_and_invalidate(req_desc, sizeof(*req_desc)); + ufshcd_cache_flush(req_desc, sizeof(*req_desc)); return; }

@@ -1487,8 +1502,8 @@ static void prepare_prdt_table(struct ufs_hba *hba, struct scsi_cmd *pccb) prepare_prdt_desc(&prd_table[table_length - i - 1], buf, datalen - 1);

req_desc->prd_table_length = table_length; - ufshcd_cache_flush_and_invalidate(prd_table, sizeof(*prd_table) * table_length); - ufshcd_cache_flush_and_invalidate(req_desc, sizeof(*req_desc)); + ufshcd_cache_flush(prd_table, sizeof(*prd_table) * table_length); + ufshcd_cache_flush(req_desc, sizeof(*req_desc)); }

static int ufs_scsi_exec(struct udevice *scsi_dev, struct scsi_cmd *pccb)

From: Marek Vasut marek.vasut+renesas@mailbox.org

Add UFSHCD_QUIRK_BROKEN_64BIT_ADDRESS for host controllers which do not support 64-bit addressing.

Ported from Linux kernel commit 6554400d6f66 ("scsi: ufs: core: Add UFSHCD_QUIRK_BROKEN_64BIT_ADDRESS") with ufs_scsi_buffer_aligned() based on U-Boot generic bounce buffer.

Signed-off-by: Marek Vasut marek.vasut+renesas@mailbox.org Tested-by: Venkatesh Yadav Abbarapu venkatesh.abbarapu@amd.com --- drivers/ufs/ufs.h | 6 ++++++ 1 file changed, 6 insertions(+)

diff --git a/drivers/ufs/ufs.h b/drivers/ufs/ufs.h index 43042c294bb..c92f47d82b5 100644 --- a/drivers/ufs/ufs.h +++ b/drivers/ufs/ufs.h @@ -732,6 +732,12 @@ struct ufs_hba { */ #define UFSHCD_QUIRK_HIBERN_FASTAUTO BIT(2)

+/* + * This quirk needs to be enabled if the host controller has + * 64-bit addressing supported capability but it doesn't work. + */ +#define UFSHCD_QUIRK_BROKEN_64BIT_ADDRESS 0x2 + /* Virtual memory reference */ struct utp_transfer_cmd_desc *ucdl; struct utp_transfer_req_desc *utrdl;

From: Bhupesh Sharma bhupesh.linux@gmail.com

Add definition of 'ufshcd_rmwl()' helper function which would be later used by Qualcomm UFS driver to read-modify-write registers.

Ported from Linux kernel commits: e785060ea3a1 ("ufs: definitions for phy interface") cff91daf52d3 ("scsi: ufs: Fix kernel-doc syntax in ufshcd.h")

Signed-off-by: Bhupesh Sharma bhupesh.sharma@linaro.org Tested-by: Venkatesh Yadav Abbarapu venkatesh.abbarapu@amd.com --- drivers/ufs/ufs.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)

diff --git a/drivers/ufs/ufs.h b/drivers/ufs/ufs.h index b55c4a9e996..555f8a6857d 100644 --- a/drivers/ufs/ufs.h +++ b/drivers/ufs/ufs.h @@ -3,6 +3,7 @@ #define __UFS_H

#include <linux/types.h> +#include <asm/io.h> #include "unipro.h"

struct udevice; @@ -933,6 +934,23 @@ enum { #define ufshcd_readl(hba, reg) \ readl((hba)->mmio_base + (reg))

+/** + * ufshcd_rmwl - perform read/modify/write for a controller register + * @hba: per adapter instance + * @mask: mask to apply on read value + * @val: actual value to write + * @reg: register address + */ +static inline void ufshcd_rmwl(struct ufs_hba *hba, u32 mask, u32 val, u32 reg) +{ + u32 tmp; + + tmp = ufshcd_readl(hba, reg); + tmp &= ~mask; + tmp |= (val & mask); + ufshcd_writel(hba, tmp, reg); +} + /* UTRLRSR - UTP Transfer Request Run-Stop Register 60h */ #define UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT 0x1

From: Bhupesh Sharma bhupesh.linux@gmail.com

Sync u-boot UFS driver to add all possible UFS Quirks as supported by Linux UFS driver as well.

Synced with include/ufs/ufshcd.h from Linux v6.11 release

Signed-off-by: Bhupesh Sharma bhupesh.sharma@linaro.org Tested-by: Venkatesh Yadav Abbarapu venkatesh.abbarapu@amd.com --- drivers/ufs/ufs.h | 193 +++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 162 insertions(+), 31 deletions(-)

diff --git a/drivers/ufs/ufs.h b/drivers/ufs/ufs.h index 555f8a6857d..a1006b80e88 100644 --- a/drivers/ufs/ufs.h +++ b/drivers/ufs/ufs.h @@ -712,38 +712,169 @@ struct ufs_hba { u32 version; u32 intr_mask; u32 quirks; -/* - * If UFS host controller is having issue in processing LCC (Line - * Control Command) coming from device then enable this quirk. - * When this quirk is enabled, host controller driver should disable - * the LCC transmission on UFS device (by clearing TX_LCC_ENABLE - * attribute of device to 0). - */ -#define UFSHCD_QUIRK_BROKEN_LCC BIT(0) - -/* - * This quirk needs to be enabled if the host controller has - * 64-bit addressing supported capability but it doesn't work. - */ -#define UFSHCD_QUIRK_BROKEN_64BIT_ADDRESS BIT(1) - -/* - * This quirk needs to be enabled if the host controller has - * auto-hibernate capability but it's FASTAUTO only. - */ -#define UFSHCD_QUIRK_HIBERN_FASTAUTO BIT(2) - -/* - * This quirk needs to be enabled if the host controller has - * 64-bit addressing supported capability but it doesn't work. - */ -#define UFSHCD_QUIRK_BROKEN_64BIT_ADDRESS 0x2

-/* - * This quirk needs to be enabled if the host controller has - * auto-hibernate capability but it's FASTAUTO only. - */ -#define UFSHCD_QUIRK_HIBERN_FASTAUTO 0x4 +enum ufshcd_quirks { + /* Interrupt aggregation support is broken */ + UFSHCD_QUIRK_BROKEN_INTR_AGGR = 1 << 0, + + /* + * delay before each dme command is required as the unipro + * layer has shown instabilities + */ + UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS = 1 << 1, + + /* + * If UFS host controller is having issue in processing LCC (Line + * Control Command) coming from device then enable this quirk. + * When this quirk is enabled, host controller driver should disable + * the LCC transmission on UFS device (by clearing TX_LCC_ENABLE + * attribute of device to 0). + */ + UFSHCD_QUIRK_BROKEN_LCC = 1 << 2, + + /* + * The attribute PA_RXHSUNTERMCAP specifies whether or not the + * inbound Link supports unterminated line in HS mode. Setting this + * attribute to 1 fixes moving to HS gear. + */ + UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP = 1 << 3, + + /* + * This quirk needs to be enabled if the host controller only allows + * accessing the peer dme attributes in AUTO mode (FAST AUTO or + * SLOW AUTO). + */ + UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE = 1 << 4, + + /* + * This quirk needs to be enabled if the host controller doesn't + * advertise the correct version in UFS_VER register. If this quirk + * is enabled, standard UFS host driver will call the vendor specific + * ops (get_ufs_hci_version) to get the correct version. + */ + UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION = 1 << 5, + + /* + * Clear handling for transfer/task request list is just opposite. + */ + UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR = 1 << 6, + + /* + * This quirk needs to be enabled if host controller doesn't allow + * that the interrupt aggregation timer and counter are reset by s/w. + */ + UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR = 1 << 7, + + /* + * This quirks needs to be enabled if host controller cannot be + * enabled via HCE register. + */ + UFSHCI_QUIRK_BROKEN_HCE = 1 << 8, + + /* + * This quirk needs to be enabled if the host controller regards + * resolution of the values of PRDTO and PRDTL in UTRD as byte. + */ + UFSHCD_QUIRK_PRDT_BYTE_GRAN = 1 << 9, + + /* + * This quirk needs to be enabled if the host controller reports + * OCS FATAL ERROR with device error through sense data + */ + UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR = 1 << 10, + + /* + * This quirk needs to be enabled if the host controller has + * auto-hibernate capability but it doesn't work. + */ + UFSHCD_QUIRK_BROKEN_AUTO_HIBERN8 = 1 << 11, + + /* + * This quirk needs to disable manual flush for write booster + */ + UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL = 1 << 12, + + /* + * This quirk needs to disable unipro timeout values + * before power mode change + */ + UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING = 1 << 13, + + /* + * This quirk needs to be enabled if the host controller does not + * support UIC command + */ + UFSHCD_QUIRK_BROKEN_UIC_CMD = 1 << 15, + + /* + * This quirk needs to be enabled if the host controller cannot + * support physical host configuration. + */ + UFSHCD_QUIRK_SKIP_PH_CONFIGURATION = 1 << 16, + + /* + * This quirk needs to be enabled if the host controller has + * 64-bit addressing supported capability but it doesn't work. + */ + UFSHCD_QUIRK_BROKEN_64BIT_ADDRESS = 1 << 17, + + /* + * This quirk needs to be enabled if the host controller has + * auto-hibernate capability but it's FASTAUTO only. + */ + UFSHCD_QUIRK_HIBERN_FASTAUTO = 1 << 18, + + /* + * This quirk needs to be enabled if the host controller needs + * to reinit the device after switching to maximum gear. + */ + UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH = 1 << 19, + + /* + * Some host raises interrupt (per queue) in addition to + * CQES (traditional) when ESI is disabled. + * Enable this quirk will disable CQES and use per queue interrupt. + */ + UFSHCD_QUIRK_MCQ_BROKEN_INTR = 1 << 20, + + /* + * Some host does not implement SQ Run Time Command (SQRTC) register + * thus need this quirk to skip related flow. + */ + UFSHCD_QUIRK_MCQ_BROKEN_RTC = 1 << 21, + + /* + * This quirk needs to be enabled if the host controller supports inline + * encryption but it needs to initialize the crypto capabilities in a + * nonstandard way and/or needs to override blk_crypto_ll_ops. If + * enabled, the standard code won't initialize the blk_crypto_profile; + * ufs_hba_variant_ops::init() must do it instead. + */ + UFSHCD_QUIRK_CUSTOM_CRYPTO_PROFILE = 1 << 22, + + /* + * This quirk needs to be enabled if the host controller supports inline + * encryption but does not support the CRYPTO_GENERAL_ENABLE bit, i.e. + * host controller initialization fails if that bit is set. + */ + UFSHCD_QUIRK_BROKEN_CRYPTO_ENABLE = 1 << 23, + + /* + * This quirk needs to be enabled if the host controller driver copies + * cryptographic keys into the PRDT in order to send them to hardware, + * and therefore the PRDT should be zeroized after each request (as per + * the standard best practice for managing keys). + */ + UFSHCD_QUIRK_KEYS_IN_PRDT = 1 << 24, + + /* + * This quirk indicates that the controller reports the value 1 (not + * supported) in the Legacy Single DoorBell Support (LSDBS) bit of the + * Controller Capabilities register although it supports the legacy + * single doorbell mode. + */ + UFSHCD_QUIRK_BROKEN_LSDBS_CAP = 1 << 25, +};

/* Virtual memory reference */ struct utp_transfer_cmd_desc *ucdl;

From: Bhupesh Sharma bhupesh.linux@gmail.com

Add missing wmb() and mb() barriers in the u-boot UFS core framework driver to allow registers updates to happen before follow-up read operations.

This makes the barrier placement similar to the Linux UFS driver, synced from the Linux v6.9 release.

Starting from the v6.10 release, the barriers were replaced with a register read-back in [1], this will ported to u-boot in a second time.

[1] https://lore.kernel.org/all/20240329-ufs-reset-ensure-effect-before-delay-v5...

Signed-off-by: Bhupesh Sharma bhupesh.sharma@linaro.org Tested-by: Venkatesh Yadav Abbarapu venkatesh.abbarapu@amd.com --- drivers/ufs/ufs.c | 11 +++++++++++ 1 file changed, 11 insertions(+)

diff --git a/drivers/ufs/ufs.c b/drivers/ufs/ufs.c index 565a6af1404..5d4e5424358 100644 --- a/drivers/ufs/ufs.c +++ b/drivers/ufs/ufs.c @@ -432,6 +432,12 @@ static int ufshcd_make_hba_operational(struct ufs_hba *hba) ufshcd_writel(hba, upper_32_bits((dma_addr_t)hba->utmrdl), REG_UTP_TASK_REQ_LIST_BASE_H);

+ /* + * Make sure base address and interrupt setup are updated before + * enabling the run/stop registers below. + */ + wmb(); + /* * UCRDY, UTMRLDY and UTRLRDY bits must be 1 */ @@ -861,6 +867,9 @@ static int ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)

ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL);

+ /* Make sure doorbell reg is updated before reading interrupt status */ + wmb(); + start = get_timer(0); do { intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS); @@ -1994,6 +2003,8 @@ int ufshcd_probe(struct udevice *ufs_dev, struct ufs_hba_ops *hba_ops) REG_INTERRUPT_STATUS); ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE);

+ mb(); + err = ufshcd_hba_enable(hba); if (err) { dev_err(hba->dev, "Host controller enable failed\n");

The link_startup_again logic was added in Linux to handle device that were set in LinkDown state, which should not be the case since U-boot doesn't set LinkDown state are init, and Linux sets the device active in ufshcd_init() for the first link startup.

ufshcd_set_ufs_dev_active(hba) is called at ufshcd_init() right before scheduling an ufshcd_async_scan that will call ufshcd_device_init() then ufshcd_link_startup().

The comment in probe says: /* * We are assuming that device wasn't put in sleep/power-down * state exclusively during the boot stage before kernel. * This assumption helps avoid doing link startup twice during * ufshcd_probe_hba(). */ we can assume the same from U-Boot.

While it worked to far, it breaks link startup for Qualcomm Controllers v5, let's just remove the logic.

Tested-by: Venkatesh Yadav Abbarapu venkatesh.abbarapu@amd.com Signed-off-by: Neil Armstrong neil.armstrong@linaro.org --- drivers/ufs/ufs.c | 8 -------- 1 file changed, 8 deletions(-)

diff --git a/drivers/ufs/ufs.c b/drivers/ufs/ufs.c index d2df5c26f76..e34e4586224 100644 --- a/drivers/ufs/ufs.c +++ b/drivers/ufs/ufs.c @@ -462,9 +462,7 @@ static int ufshcd_link_startup(struct ufs_hba *hba) { int ret; int retries = DME_LINKSTARTUP_RETRIES; - bool link_startup_again = true;

-link_startup: do { ufshcd_ops_link_startup_notify(hba, PRE_CHANGE);

@@ -490,12 +488,6 @@ link_startup: /* failed to get the link up... retire */ goto out;

- if (link_startup_again) { - link_startup_again = false; - retries = DME_LINKSTARTUP_RETRIES; - goto link_startup; - } - /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */ ufshcd_init_pwr_info(hba);