Use an enum to differentiate the type of I/O (reading or writing a page). Also update the request iterator.

This is a port of linux patch 701981cab01696584a12e5f0e7c2ad931a326059 created by Miquel Raynal miquel.raynal@bootlin.com

Signed-off-by: Mikhail Kshevetskiy mikhail.kshevetskiy@iopsys.eu --- drivers/mtd/nand/spi/core.c | 4 ++-- include/linux/mtd/nand.h | 18 ++++++++++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/drivers/mtd/nand/spi/core.c b/drivers/mtd/nand/spi/core.c index ea00cd7dcf0..8f227ce81fa 100644 --- a/drivers/mtd/nand/spi/core.c +++ b/drivers/mtd/nand/spi/core.c @@ -523,7 +523,7 @@ static int spinand_mtd_read(struct mtd_info *mtd, loff_t from, mutex_lock(&spinand->lock); #endif

- nanddev_io_for_each_page(nand, from, ops, &iter) { + nanddev_io_for_each_page(nand, NAND_PAGE_READ, from, ops, &iter) { schedule(); ret = spinand_select_target(spinand, iter.req.pos.target); if (ret) @@ -575,7 +575,7 @@ static int spinand_mtd_write(struct mtd_info *mtd, loff_t to, mutex_lock(&spinand->lock); #endif

- nanddev_io_for_each_page(nand, to, ops, &iter) { + nanddev_io_for_each_page(nand, NAND_PAGE_WRITE, to, ops, &iter) { schedule(); ret = spinand_select_target(spinand, iter.req.pos.target); if (ret) diff --git a/include/linux/mtd/nand.h b/include/linux/mtd/nand.h index 651f8706df5..0afdaed5715 100644 --- a/include/linux/mtd/nand.h +++ b/include/linux/mtd/nand.h @@ -80,8 +80,19 @@ struct nand_pos { unsigned int page; };

+/** + * enum nand_page_io_req_type - Direction of an I/O request + * @NAND_PAGE_READ: from the chip, to the controller + * @NAND_PAGE_WRITE: from the controller, to the chip + */ +enum nand_page_io_req_type { + NAND_PAGE_READ = 0, + NAND_PAGE_WRITE, +}; + /** * struct nand_page_io_req - NAND I/O request object + * @type: the type of page I/O: read or write * @pos: the position this I/O request is targeting * @dataoffs: the offset within the page * @datalen: number of data bytes to read from/write to this page @@ -97,6 +108,7 @@ struct nand_pos { * specific commands/operations. */ struct nand_page_io_req { + enum nand_page_io_req_type type; struct nand_pos pos; unsigned int dataoffs; unsigned int datalen; @@ -613,11 +625,13 @@ static inline void nanddev_pos_next_page(struct nand_device *nand, * layer. */ static inline void nanddev_io_iter_init(struct nand_device *nand, + enum nand_page_io_req_type reqtype, loff_t offs, struct mtd_oob_ops *req, struct nand_io_iter *iter) { struct mtd_info *mtd = nanddev_to_mtd(nand);

+ iter->req.type = reqtype; iter->req.mode = req->mode; iter->req.dataoffs = nanddev_offs_to_pos(nand, offs, &iter->req.pos); iter->req.ooboffs = req->ooboffs; @@ -687,8 +701,8 @@ static inline bool nanddev_io_iter_end(struct nand_device *nand, * * Should be used for iterate over pages that are contained in an MTD request. */ -#define nanddev_io_for_each_page(nand, start, req, iter) \ - for (nanddev_io_iter_init(nand, start, req, iter); \ +#define nanddev_io_for_each_page(nand, type, start, req, iter) \ + for (nanddev_io_iter_init(nand, type, start, req, iter); \ !nanddev_io_iter_end(nand, iter); \ nanddev_io_iter_next_page(nand, iter))

also call schedule() to allow periodic actions

Signed-off-by: Mikhail Kshevetskiy mikhail.kshevetskiy@iopsys.eu --- drivers/mtd/nand/spi/core.c | 35 ++++++++++++++++++++++++++++------- include/linux/mtd/spinand.h | 22 ++++++++++++++++++++++ 2 files changed, 50 insertions(+), 7 deletions(-)

diff --git a/drivers/mtd/nand/spi/core.c b/drivers/mtd/nand/spi/core.c index 62779dd3e51..a10605487f3 100644 --- a/drivers/mtd/nand/spi/core.c +++ b/drivers/mtd/nand/spi/core.c @@ -32,6 +32,7 @@ #include <linux/bug.h> #include <linux/mtd/spinand.h> #include <linux/printk.h> +#include <linux/delay.h> #endif

struct spinand_plat { @@ -362,21 +363,29 @@ static int spinand_erase_op(struct spinand_device *spinand, return spi_mem_exec_op(spinand->slave, &op); }

-static int spinand_wait(struct spinand_device *spinand, u8 *s) +static int spinand_wait(struct spinand_device *spinand, + unsigned long initial_delay_us, + unsigned long poll_delay_us, + u8 *s) { unsigned long start, stop; u8 status; int ret;

+ udelay(initial_delay_us); start = get_timer(0); - stop = 400; + stop = SPINAND_WAITRDY_TIMEOUT_MS; do { + schedule(); + ret = spinand_read_status(spinand, &status); if (ret) return ret;

if (!(status & STATUS_BUSY)) goto out; + + udelay(poll_delay_us); } while (get_timer(start) < stop);

/* @@ -418,7 +427,10 @@ static int spinand_reset_op(struct spinand_device *spinand) if (ret) return ret;

- return spinand_wait(spinand, NULL); + return spinand_wait(spinand, + SPINAND_RESET_INITIAL_DELAY_US, + SPINAND_RESET_POLL_DELAY_US, + NULL); }

static int spinand_lock_block(struct spinand_device *spinand, u8 lock) @@ -466,7 +478,10 @@ static int spinand_read_page(struct spinand_device *spinand, if (ret) return ret;

- ret = spinand_wait(spinand, &status); + ret = spinand_wait(spinand, + SPINAND_READ_INITIAL_DELAY_US, + SPINAND_READ_POLL_DELAY_US, + &status); if (ret < 0) return ret;

@@ -498,9 +513,12 @@ static int spinand_write_page(struct spinand_device *spinand, if (ret) return ret;

- ret = spinand_wait(spinand, &status); + ret = spinand_wait(spinand, + SPINAND_WRITE_INITIAL_DELAY_US, + SPINAND_WRITE_POLL_DELAY_US, + &status); if (!ret && (status & STATUS_PROG_FAILED)) - ret = -EIO; + return -EIO;

return ret; } @@ -702,7 +720,10 @@ static int spinand_erase(struct nand_device *nand, const struct nand_pos *pos) if (ret) return ret;

- ret = spinand_wait(spinand, &status); + ret = spinand_wait(spinand, + SPINAND_ERASE_INITIAL_DELAY_US, + SPINAND_ERASE_POLL_DELAY_US, + &status); if (!ret && (status & STATUS_ERASE_FAILED)) ret = -EIO;

diff --git a/include/linux/mtd/spinand.h b/include/linux/mtd/spinand.h index 5934b7604cc..b701d25f73d 100644 --- a/include/linux/mtd/spinand.h +++ b/include/linux/mtd/spinand.h @@ -176,6 +176,28 @@ struct spinand_op; struct spinand_device;

#define SPINAND_MAX_ID_LEN 4 +/* + * For erase, write and read operation, we got the following timings : + * tBERS (erase) 1ms to 4ms + * tPROG 300us to 400us + * tREAD 25us to 100us + * In order to minimize latency, the min value is divided by 4 for the + * initial delay, and dividing by 20 for the poll delay. + * For reset, 5us/10us/500us if the device is respectively + * reading/programming/erasing when the RESET occurs. Since we always + * issue a RESET when the device is IDLE, 5us is selected for both initial + * and poll delay. + */ +#define SPINAND_READ_INITIAL_DELAY_US 6 +#define SPINAND_READ_POLL_DELAY_US 5 +#define SPINAND_RESET_INITIAL_DELAY_US 5 +#define SPINAND_RESET_POLL_DELAY_US 5 +#define SPINAND_WRITE_INITIAL_DELAY_US 75 +#define SPINAND_WRITE_POLL_DELAY_US 15 +#define SPINAND_ERASE_INITIAL_DELAY_US 250 +#define SPINAND_ERASE_POLL_DELAY_US 50 + +#define SPINAND_WAITRDY_TIMEOUT_MS 400

/** * struct spinand_id - SPI NAND id structure

Signed-off-by: Mikhail Kshevetskiy mikhail.kshevetskiy@iopsys.eu --- drivers/mtd/nand/spi/core.c | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/drivers/mtd/nand/spi/core.c b/drivers/mtd/nand/spi/core.c index b58d9e00907..9629fac3388 100644 --- a/drivers/mtd/nand/spi/core.c +++ b/drivers/mtd/nand/spi/core.c @@ -530,12 +530,12 @@ static int spinand_mtd_read(struct mtd_info *mtd, loff_t from, struct nand_device *nand = mtd_to_nanddev(mtd); unsigned int max_bitflips = 0; struct nand_io_iter iter; - bool enable_ecc = false; + bool disable_ecc = false; bool ecc_failed = false; int ret = 0;

- if (ops->mode != MTD_OPS_RAW && spinand->eccinfo.ooblayout) - enable_ecc = true; + if (ops->mode == MTD_OPS_RAW || !spinand->eccinfo.ooblayout) + disable_ecc = true;

#ifndef __UBOOT__ mutex_lock(&spinand->lock); @@ -543,15 +543,18 @@ static int spinand_mtd_read(struct mtd_info *mtd, loff_t from,

nanddev_io_for_each_page(nand, NAND_PAGE_READ, from, ops, &iter) { schedule(); + if (disable_ecc) + iter.req.mode = MTD_OPS_RAW; + ret = spinand_select_target(spinand, iter.req.pos.target); if (ret) break;

- ret = spinand_ecc_enable(spinand, enable_ecc); + ret = spinand_ecc_enable(spinand, !disable_ecc); if (ret) break;

- ret = spinand_read_page(spinand, &iter.req, enable_ecc); + ret = spinand_read_page(spinand, &iter.req, !disable_ecc); if (ret < 0 && ret != -EBADMSG) break;

@@ -583,11 +586,11 @@ static int spinand_mtd_write(struct mtd_info *mtd, loff_t to, struct spinand_device *spinand = mtd_to_spinand(mtd); struct nand_device *nand = mtd_to_nanddev(mtd); struct nand_io_iter iter; - bool enable_ecc = false; + bool disable_ecc = false; int ret = 0;

- if (ops->mode != MTD_OPS_RAW && mtd->ooblayout) - enable_ecc = true; + if (ops->mode == MTD_OPS_RAW || !mtd->ooblayout) + disable_ecc = true;

#ifndef __UBOOT__ mutex_lock(&spinand->lock); @@ -595,11 +598,14 @@ static int spinand_mtd_write(struct mtd_info *mtd, loff_t to,

nanddev_io_for_each_page(nand, NAND_PAGE_WRITE, to, ops, &iter) { schedule(); + if (disable_ecc) + iter.req.mode = MTD_OPS_RAW; + ret = spinand_select_target(spinand, iter.req.pos.target); if (ret) break;

- ret = spinand_ecc_enable(spinand, enable_ecc); + ret = spinand_ecc_enable(spinand, !disable_ecc); if (ret) break;

changes: * Move spinand_check_ecc_status(), spinand_noecc_ooblayout_ecc(), spinand_noecc_ooblayout_free() and spinand_noecc_ooblayout close to each other. * some code formatting * remove comments not present in linux driver

This make code more close to linux-6.10 kernel driver

Signed-off-by: Mikhail Kshevetskiy mikhail.kshevetskiy@iopsys.eu --- drivers/mtd/nand/spi/core.c | 115 +++++++++++++++++------------------- 1 file changed, 55 insertions(+), 60 deletions(-)

diff --git a/drivers/mtd/nand/spi/core.c b/drivers/mtd/nand/spi/core.c index 6ca8b7c80cc..548a7144ee3 100644 --- a/drivers/mtd/nand/spi/core.c +++ b/drivers/mtd/nand/spi/core.c @@ -222,6 +222,59 @@ static int spinand_ecc_enable(struct spinand_device *spinand, enable ? CFG_ECC_ENABLE : 0); }

+static int spinand_check_ecc_status(struct spinand_device *spinand, u8 status) +{ + struct nand_device *nand = spinand_to_nand(spinand); + + if (spinand->eccinfo.get_status) + return spinand->eccinfo.get_status(spinand, status); + + switch (status & STATUS_ECC_MASK) { + case STATUS_ECC_NO_BITFLIPS: + return 0; + + case STATUS_ECC_HAS_BITFLIPS: + /* + * We have no way to know exactly how many bitflips have been + * fixed, so let's return the maximum possible value so that + * wear-leveling layers move the data immediately. + */ + return nand->eccreq.strength; + + case STATUS_ECC_UNCOR_ERROR: + return -EBADMSG; + + default: + break; + } + + return -EINVAL; +} + +static int spinand_noecc_ooblayout_ecc(struct mtd_info *mtd, int section, + struct mtd_oob_region *region) +{ + return -ERANGE; +} + +static int spinand_noecc_ooblayout_free(struct mtd_info *mtd, int section, + struct mtd_oob_region *region) +{ + if (section) + return -ERANGE; + + /* Reserve 2 bytes for the BBM. */ + region->offset = 2; + region->length = 62; + + return 0; +} + +static const struct mtd_ooblayout_ops spinand_noecc_ooblayout = { + .ecc = spinand_noecc_ooblayout_ecc, + .rfree = spinand_noecc_ooblayout_free, +}; + static int spinand_write_enable_op(struct spinand_device *spinand) { struct spi_mem_op op = SPINAND_WR_EN_DIS_OP(true); @@ -413,9 +466,8 @@ out: static int spinand_read_id_op(struct spinand_device *spinand, u8 naddr, u8 ndummy, u8 *buf) { - struct spi_mem_op op = SPINAND_READID_OP(naddr, ndummy, - spinand->scratchbuf, - SPINAND_MAX_ID_LEN); + struct spi_mem_op op = SPINAND_READID_OP( + naddr, ndummy, spinand->scratchbuf, SPINAND_MAX_ID_LEN); int ret;

ret = spi_mem_exec_op(spinand->slave, &op); @@ -445,35 +497,6 @@ static int spinand_lock_block(struct spinand_device *spinand, u8 lock) return spinand_write_reg_op(spinand, REG_BLOCK_LOCK, lock); }

-static int spinand_check_ecc_status(struct spinand_device *spinand, u8 status) -{ - struct nand_device *nand = spinand_to_nand(spinand); - - if (spinand->eccinfo.get_status) - return spinand->eccinfo.get_status(spinand, status); - - switch (status & STATUS_ECC_MASK) { - case STATUS_ECC_NO_BITFLIPS: - return 0; - - case STATUS_ECC_HAS_BITFLIPS: - /* - * We have no way to know exactly how many bitflips have been - * fixed, so let's return the maximum possible value so that - * wear-leveling layers move the data immediately. - */ - return nand->eccreq.strength; - - case STATUS_ECC_UNCOR_ERROR: - return -EBADMSG; - - default: - break; - } - - return -EINVAL; -} - static int spinand_read_page(struct spinand_device *spinand, const struct nand_page_io_req *req, bool ecc_enabled) @@ -1056,30 +1079,6 @@ static int spinand_detect(struct spinand_device *spinand) return 0; }

-static int spinand_noecc_ooblayout_ecc(struct mtd_info *mtd, int section, - struct mtd_oob_region *region) -{ - return -ERANGE; -} - -static int spinand_noecc_ooblayout_free(struct mtd_info *mtd, int section, - struct mtd_oob_region *region) -{ - if (section) - return -ERANGE; - - /* Reserve 2 bytes for the BBM. */ - region->offset = 2; - region->length = 62; - - return 0; -} - -static const struct mtd_ooblayout_ops spinand_noecc_ooblayout = { - .ecc = spinand_noecc_ooblayout_ecc, - .rfree = spinand_noecc_ooblayout_free, -}; - static int spinand_init_flash(struct spinand_device *spinand) { struct udevice *dev = spinand->slave->dev; @@ -1169,10 +1168,6 @@ static int spinand_init(struct spinand_device *spinand) if (ret) goto err_manuf_cleanup;

- /* - * Right now, we don't support ECC, so let the whole oob - * area is available for user. - */ mtd->_read_oob = spinand_mtd_read; mtd->_write_oob = spinand_mtd_write; mtd->_block_isbad = spinand_mtd_block_isbad;

only spinand on_die ecc is supported for a moment

Signed-off-by: Mikhail Kshevetskiy mikhail.kshevetskiy@iopsys.eu --- drivers/mtd/nand/Makefile | 2 +- drivers/mtd/nand/core.c | 130 +++++++++++++++- drivers/mtd/nand/ecc.c | 249 ++++++++++++++++++++++++++++++ drivers/mtd/nand/spi/core.c | 207 ++++++++++++++++++++----- drivers/mtd/nand/spi/foresee.c | 2 +- drivers/mtd/nand/spi/macronix.c | 7 +- drivers/mtd/nand/spi/micron.c | 2 +- drivers/mtd/nand/spi/toshiba.c | 10 +- drivers/mtd/nand/spi/winbond.c | 10 +- include/linux/mtd/nand.h | 261 ++++++++++++++++++++++++++++++-- include/linux/mtd/spinand.h | 13 +- include/spi-mem.h | 2 + 12 files changed, 830 insertions(+), 65 deletions(-) create mode 100644 drivers/mtd/nand/ecc.c

diff --git a/drivers/mtd/nand/Makefile b/drivers/mtd/nand/Makefile index 96e186600a1..56179188e92 100644 --- a/drivers/mtd/nand/Makefile +++ b/drivers/mtd/nand/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0+

ifeq ($(CONFIG_SPL_BUILD)$(CONFIG_TPL_BUILD),) -nandcore-objs := core.o bbt.o +nandcore-objs := core.o bbt.o ecc.o obj-$(CONFIG_MTD_NAND_CORE) += nandcore.o obj-$(CONFIG_MTD_RAW_NAND) += raw/ obj-$(CONFIG_MTD_SPI_NAND) += spi/ diff --git a/drivers/mtd/nand/core.c b/drivers/mtd/nand/core.c index 472ad0bdefb..6c90d576de3 100644 --- a/drivers/mtd/nand/core.c +++ b/drivers/mtd/nand/core.c @@ -129,7 +129,7 @@ EXPORT_SYMBOL_GPL(nanddev_isreserved); * * Return: 0 in case of success, a negative error code otherwise. */ -static int nanddev_erase(struct nand_device *nand, const struct nand_pos *pos) +int nanddev_erase(struct nand_device *nand, const struct nand_pos *pos) { unsigned int entry;

@@ -187,6 +187,134 @@ int nanddev_mtd_erase(struct mtd_info *mtd, struct erase_info *einfo) } EXPORT_SYMBOL_GPL(nanddev_mtd_erase);

+/** + * nanddev_get_ecc_engine() - Find and get a suitable ECC engine + * @nand: NAND device + */ +static int nanddev_get_ecc_engine(struct nand_device *nand) +{ + int engine_type; + + /* Read the user desires in terms of ECC engine/configuration */ + of_get_nand_ecc_user_config(nand); + + engine_type = nand->ecc.user_conf.engine_type; + if (engine_type == NAND_ECC_ENGINE_TYPE_INVALID) + engine_type = nand->ecc.defaults.engine_type; + + switch (engine_type) { + case NAND_ECC_ENGINE_TYPE_NONE: + return 0; + case NAND_ECC_ENGINE_TYPE_SOFT: + nand->ecc.engine = nand_ecc_get_sw_engine(nand); + break; + case NAND_ECC_ENGINE_TYPE_ON_DIE: + nand->ecc.engine = nand_ecc_get_on_die_hw_engine(nand); + break; + case NAND_ECC_ENGINE_TYPE_ON_HOST: + nand->ecc.engine = nand_ecc_get_on_host_hw_engine(nand); + if (PTR_ERR(nand->ecc.engine) == -EPROBE_DEFER) + return -EPROBE_DEFER; + break; + default: + pr_err("Missing ECC engine type\n"); + } + + if (!nand->ecc.engine) + return -EINVAL; + + return 0; +} + +/** + * nanddev_put_ecc_engine() - Dettach and put the in-use ECC engine + * @nand: NAND device + */ +static int nanddev_put_ecc_engine(struct nand_device *nand) +{ + switch (nand->ecc.ctx.conf.engine_type) { + case NAND_ECC_ENGINE_TYPE_ON_HOST: + nand_ecc_put_on_host_hw_engine(nand); + break; + case NAND_ECC_ENGINE_TYPE_NONE: + case NAND_ECC_ENGINE_TYPE_SOFT: + case NAND_ECC_ENGINE_TYPE_ON_DIE: + default: + break; + } + + return 0; +} + +/** + * nanddev_find_ecc_configuration() - Find a suitable ECC configuration + * @nand: NAND device + */ +static int nanddev_find_ecc_configuration(struct nand_device *nand) +{ + int ret; + + if (!nand->ecc.engine) + return -ENOTSUPP; + + ret = nand_ecc_init_ctx(nand); + if (ret) + return ret; + + if (!nand_ecc_is_strong_enough(nand)) + pr_warn("WARNING: %s: the ECC used on your system is too weak compared to the one required by the NAND chip\n", + nand->mtd->name); + + return 0; +} + +/** + * nanddev_ecc_engine_init() - Initialize an ECC engine for the chip + * @nand: NAND device + */ +int nanddev_ecc_engine_init(struct nand_device *nand) +{ + int ret; + + /* Look for the ECC engine to use */ + ret = nanddev_get_ecc_engine(nand); + if (ret) { + if (ret != -EPROBE_DEFER) + pr_err("No ECC engine found\n"); + + return ret; + } + + /* No ECC engine requested */ + if (!nand->ecc.engine) + return 0; + + /* Configure the engine: balance user input and chip requirements */ + ret = nanddev_find_ecc_configuration(nand); + if (ret) { + pr_err("No suitable ECC configuration\n"); + nanddev_put_ecc_engine(nand); + + return ret; + } + + return 0; +} +EXPORT_SYMBOL_GPL(nanddev_ecc_engine_init); + +/** + * nanddev_ecc_engine_cleanup() - Cleanup ECC engine initializations + * @nand: NAND device + */ +void nanddev_ecc_engine_cleanup(struct nand_device *nand) +{ + if (nand->ecc.engine) + nand_ecc_cleanup_ctx(nand); + + nanddev_put_ecc_engine(nand); +} +EXPORT_SYMBOL_GPL(nanddev_ecc_engine_cleanup); + /** * nanddev_init() - Initialize a NAND device * @nand: NAND device diff --git a/drivers/mtd/nand/ecc.c b/drivers/mtd/nand/ecc.c new file mode 100644 index 00000000000..58cbe7deaac --- /dev/null +++ b/drivers/mtd/nand/ecc.c @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Generic Error-Correcting Code (ECC) engine + * + * Copyright (C) 2019 Macronix + * Author: + * Miquèl RAYNAL miquel.raynal@bootlin.com + * + * + * This file describes the abstraction of any NAND ECC engine. It has been + * designed to fit most cases, including parallel NANDs and SPI-NANDs. + * + * There are three main situations where instantiating this ECC engine makes + * sense: + * - external: The ECC engine is outside the NAND pipeline, typically this + * is a software ECC engine, or an hardware engine that is + * outside the NAND controller pipeline. + * - pipelined: The ECC engine is inside the NAND pipeline, ie. on the + * controller's side. This is the case of most of the raw NAND + * controllers. In the pipeline case, the ECC bytes are + * generated/data corrected on the fly when a page is + * written/read. + * - ondie: The ECC engine is inside the NAND pipeline, on the chip's side. + * Some NAND chips can correct themselves the data. + * + * Besides the initial setup and final cleanups, the interfaces are rather + * simple: + * - prepare: Prepare an I/O request. Enable/disable the ECC engine based on + * the I/O request type. In case of software correction or external + * engine, this step may involve to derive the ECC bytes and place + * them in the OOB area before a write. + * - finish: Finish an I/O request. Correct the data in case of a read + * request and report the number of corrected bits/uncorrectable + * errors. Most likely empty for write operations, unless you have + * hardware specific stuff to do, like shutting down the engine to + * save power. + * + * The I/O request should be enclosed in a prepare()/finish() pair of calls + * and will behave differently depending on the requested I/O type: + * - raw: Correction disabled + * - ecc: Correction enabled + * + * The request direction is impacting the logic as well: + * - read: Load data from the NAND chip + * - write: Store data in the NAND chip + * + * Mixing all this combinations together gives the following behavior. + * Those are just examples, drivers are free to add custom steps in their + * prepare/finish hook. + * + * [external ECC engine] + * - external + prepare + raw + read: do nothing + * - external + finish + raw + read: do nothing + * - external + prepare + raw + write: do nothing + * - external + finish + raw + write: do nothing + * - external + prepare + ecc + read: do nothing + * - external + finish + ecc + read: calculate expected ECC bytes, extract + * ECC bytes from OOB buffer, correct + * and report any bitflip/error + * - external + prepare + ecc + write: calculate ECC bytes and store them at + * the right place in the OOB buffer based + * on the OOB layout + * - external + finish + ecc + write: do nothing + * + * [pipelined ECC engine] + * - pipelined + prepare + raw + read: disable the controller's ECC engine if + * activated + * - pipelined + finish + raw + read: do nothing + * - pipelined + prepare + raw + write: disable the controller's ECC engine if + * activated + * - pipelined + finish + raw + write: do nothing + * - pipelined + prepare + ecc + read: enable the controller's ECC engine if + * deactivated + * - pipelined + finish + ecc + read: check the status, report any + * error/bitflip + * - pipelined + prepare + ecc + write: enable the controller's ECC engine if + * deactivated + * - pipelined + finish + ecc + write: do nothing + * + * [ondie ECC engine] + * - ondie + prepare + raw + read: send commands to disable the on-chip ECC + * engine if activated + * - ondie + finish + raw + read: do nothing + * - ondie + prepare + raw + write: send commands to disable the on-chip ECC + * engine if activated + * - ondie + finish + raw + write: do nothing + * - ondie + prepare + ecc + read: send commands to enable the on-chip ECC + * engine if deactivated + * - ondie + finish + ecc + read: send commands to check the status, report + * any error/bitflip + * - ondie + prepare + ecc + write: send commands to enable the on-chip ECC + * engine if deactivated + * - ondie + finish + ecc + write: do nothing + */ + +#ifndef __UBOOT__ +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/of_platform.h> +#endif +#include <linux/mtd/nand.h> + +/** + * nand_ecc_init_ctx - Init the ECC engine context + * @nand: the NAND device + * + * On success, the caller is responsible of calling @nand_ecc_cleanup_ctx(). + */ +int nand_ecc_init_ctx(struct nand_device *nand) +{ + if (!nand->ecc.engine || !nand->ecc.engine->ops->init_ctx) + return 0; + + return nand->ecc.engine->ops->init_ctx(nand); +} +EXPORT_SYMBOL(nand_ecc_init_ctx); + +/** + * nand_ecc_cleanup_ctx - Cleanup the ECC engine context + * @nand: the NAND device + */ +void nand_ecc_cleanup_ctx(struct nand_device *nand) +{ + if (nand->ecc.engine && nand->ecc.engine->ops->cleanup_ctx) + nand->ecc.engine->ops->cleanup_ctx(nand); +} +EXPORT_SYMBOL(nand_ecc_cleanup_ctx); + +/** + * nand_ecc_prepare_io_req - Prepare an I/O request + * @nand: the NAND device + * @req: the I/O request + */ +int nand_ecc_prepare_io_req(struct nand_device *nand, + struct nand_page_io_req *req) +{ + if (!nand->ecc.engine || !nand->ecc.engine->ops->prepare_io_req) + return 0; + + return nand->ecc.engine->ops->prepare_io_req(nand, req); +} +EXPORT_SYMBOL(nand_ecc_prepare_io_req); + +/** + * nand_ecc_finish_io_req - Finish an I/O request + * @nand: the NAND device + * @req: the I/O request + */ +int nand_ecc_finish_io_req(struct nand_device *nand, + struct nand_page_io_req *req) +{ + if (!nand->ecc.engine || !nand->ecc.engine->ops->finish_io_req) + return 0; + + return nand->ecc.engine->ops->finish_io_req(nand, req); +} +EXPORT_SYMBOL(nand_ecc_finish_io_req); + +void of_get_nand_ecc_user_config(struct nand_device *nand) +{ + nand->ecc.user_conf.engine_type = NAND_ECC_ENGINE_TYPE_ON_DIE; + nand->ecc.user_conf.algo = NAND_ECC_ALGO_UNKNOWN; + nand->ecc.user_conf.placement = NAND_ECC_PLACEMENT_UNKNOWN; +} +EXPORT_SYMBOL(of_get_nand_ecc_user_config); + +/** + * nand_ecc_is_strong_enough - Check if the chip configuration meets the + * datasheet requirements. + * + * @nand: Device to check + * + * If our configuration corrects A bits per B bytes and the minimum + * required correction level is X bits per Y bytes, then we must ensure + * both of the following are true: + * + * (1) A / B >= X / Y + * (2) A >= X + * + * Requirement (1) ensures we can correct for the required bitflip density. + * Requirement (2) ensures we can correct even when all bitflips are clumped + * in the same sector. + */ +bool nand_ecc_is_strong_enough(struct nand_device *nand) +{ + const struct nand_ecc_props *reqs = nanddev_get_ecc_requirements(nand); + const struct nand_ecc_props *conf = nanddev_get_ecc_conf(nand); + struct mtd_info *mtd = nanddev_to_mtd(nand); + int corr, ds_corr; + + if (conf->step_size == 0 || reqs->step_size == 0) + /* Not enough information */ + return true; + + /* + * We get the number of corrected bits per page to compare + * the correction density. + */ + corr = (mtd->writesize * conf->strength) / conf->step_size; + ds_corr = (mtd->writesize * reqs->strength) / reqs->step_size; + + return corr >= ds_corr && conf->strength >= reqs->strength; +} +EXPORT_SYMBOL(nand_ecc_is_strong_enough); + +struct nand_ecc_engine *nand_ecc_get_sw_engine(struct nand_device *nand) +{ + unsigned int algo = nand->ecc.user_conf.algo; + + if (algo == NAND_ECC_ALGO_UNKNOWN) + algo = nand->ecc.defaults.algo; + + switch (algo) { + case NAND_ECC_ALGO_HAMMING: + return nand_ecc_sw_hamming_get_engine(); + case NAND_ECC_ALGO_BCH: + return nand_ecc_sw_bch_get_engine(); + default: + break; + } + + return NULL; +} +EXPORT_SYMBOL(nand_ecc_get_sw_engine); + +struct nand_ecc_engine *nand_ecc_get_on_die_hw_engine(struct nand_device *nand) +{ + return nand->ecc.ondie_engine; +} +EXPORT_SYMBOL(nand_ecc_get_on_die_hw_engine); + +struct nand_ecc_engine *nand_ecc_get_on_host_hw_engine(struct nand_device *nand) +{ + return NULL; +} +EXPORT_SYMBOL(nand_ecc_get_on_host_hw_engine); + +void nand_ecc_put_on_host_hw_engine(struct nand_device *nand) +{ +} +EXPORT_SYMBOL(nand_ecc_put_on_host_hw_engine); + +#ifndef __UBOOT__ +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Miquel Raynal miquel.raynal@bootlin.com"); +MODULE_DESCRIPTION("Generic ECC engine"); +#endif /* __UBOOT__ */ diff --git a/drivers/mtd/nand/spi/core.c b/drivers/mtd/nand/spi/core.c index d5cb9026246..70f07be06b0 100644 --- a/drivers/mtd/nand/spi/core.c +++ b/drivers/mtd/nand/spi/core.c @@ -239,7 +239,7 @@ static int spinand_check_ecc_status(struct spinand_device *spinand, u8 status) * fixed, so let's return the maximum possible value so that * wear-leveling layers move the data immediately. */ - return nand->eccreq.strength; + return nanddev_get_ecc_conf(nand)->strength;

case STATUS_ECC_UNCOR_ERROR: return -EBADMSG; @@ -275,6 +275,92 @@ static const struct mtd_ooblayout_ops spinand_noecc_ooblayout = { .rfree = spinand_noecc_ooblayout_free, };

+static int spinand_ondie_ecc_init_ctx(struct nand_device *nand) +{ + struct spinand_device *spinand = nand_to_spinand(nand); + struct mtd_info *mtd = nanddev_to_mtd(nand); + struct spinand_ondie_ecc_conf *engine_conf; + + nand->ecc.ctx.conf.engine_type = NAND_ECC_ENGINE_TYPE_ON_DIE; + nand->ecc.ctx.conf.step_size = nand->ecc.requirements.step_size; + nand->ecc.ctx.conf.strength = nand->ecc.requirements.strength; + + engine_conf = kzalloc(sizeof(*engine_conf), GFP_KERNEL); + if (!engine_conf) + return -ENOMEM; + + nand->ecc.ctx.priv = engine_conf; + + if (spinand->eccinfo.ooblayout) + mtd_set_ooblayout(mtd, spinand->eccinfo.ooblayout); + else + mtd_set_ooblayout(mtd, &spinand_noecc_ooblayout); + + return 0; +} + +static void spinand_ondie_ecc_cleanup_ctx(struct nand_device *nand) +{ + kfree(nand->ecc.ctx.priv); +} + +static int spinand_ondie_ecc_prepare_io_req(struct nand_device *nand, + struct nand_page_io_req *req) +{ + struct spinand_device *spinand = nand_to_spinand(nand); + bool enable = (req->mode != MTD_OPS_RAW); + + memset(spinand->oobbuf, 0xff, nanddev_per_page_oobsize(nand)); + + /* Only enable or disable the engine */ + return spinand_ecc_enable(spinand, enable); +} + +static int spinand_ondie_ecc_finish_io_req(struct nand_device *nand, + struct nand_page_io_req *req) +{ + struct spinand_ondie_ecc_conf *engine_conf = nand->ecc.ctx.priv; + struct spinand_device *spinand = nand_to_spinand(nand); + struct mtd_info *mtd = spinand_to_mtd(spinand); + int ret; + + if (req->mode == MTD_OPS_RAW) + return 0; + + /* Nothing to do when finishing a page write */ + if (req->type == NAND_PAGE_WRITE) + return 0; + + /* Finish a page read: check the status, report errors/bitflips */ + ret = spinand_check_ecc_status(spinand, engine_conf->status); + if (ret == -EBADMSG) + mtd->ecc_stats.failed++; + else if (ret > 0) + mtd->ecc_stats.corrected += ret; + + return ret; +} + +static struct nand_ecc_engine_ops spinand_ondie_ecc_engine_ops = { + .init_ctx = spinand_ondie_ecc_init_ctx, + .cleanup_ctx = spinand_ondie_ecc_cleanup_ctx, + .prepare_io_req = spinand_ondie_ecc_prepare_io_req, + .finish_io_req = spinand_ondie_ecc_finish_io_req, +}; + +static struct nand_ecc_engine spinand_ondie_ecc_engine = { + .ops = &spinand_ondie_ecc_engine_ops, +}; + +static void spinand_ondie_ecc_save_status(struct nand_device *nand, u8 status) +{ + struct spinand_ondie_ecc_conf *engine_conf = nand->ecc.ctx.priv; + + if (nand->ecc.ctx.conf.engine_type == NAND_ECC_ENGINE_TYPE_ON_DIE && + engine_conf) + engine_conf->status = status; +} + static int spinand_write_enable_op(struct spinand_device *spinand) { struct spi_mem_op op = SPINAND_WR_EN_DIS_OP(true); @@ -317,7 +403,10 @@ static int spinand_read_from_cache_op(struct spinand_device *spinand, } }

- rdesc = spinand->dirmaps[req->pos.plane].rdesc; + if (req->mode == MTD_OPS_RAW) + rdesc = spinand->dirmaps[req->pos.plane].rdesc; + else + rdesc = spinand->dirmaps[req->pos.plane].rdesc_ecc;

while (nbytes) { ret = spi_mem_dirmap_read(rdesc, column, nbytes, buf); @@ -366,9 +455,12 @@ static int spinand_write_to_cache_op(struct spinand_device *spinand, * must fill the page cache entirely even if we only want to program * the data portion of the page, otherwise we might corrupt the BBM or * user data previously programmed in OOB area. + * + * Only reset the data buffer manually, the OOB buffer is prepared by + * ECC engines ->prepare_io_req() callback. */ nbytes = nanddev_page_size(nand) + nanddev_per_page_oobsize(nand); - memset(spinand->databuf, 0xff, nbytes); + memset(spinand->databuf, 0xff, nanddev_page_size(nand));

if (req->datalen) memcpy(spinand->databuf + req->dataoffs, req->databuf.out, @@ -385,7 +477,10 @@ static int spinand_write_to_cache_op(struct spinand_device *spinand, req->ooblen); }

- wdesc = spinand->dirmaps[req->pos.plane].wdesc; + if (req->mode == MTD_OPS_RAW) + wdesc = spinand->dirmaps[req->pos.plane].wdesc; + else + wdesc = spinand->dirmaps[req->pos.plane].wdesc_ecc;

while (nbytes) { ret = spi_mem_dirmap_write(wdesc, column, nbytes, buf); @@ -498,12 +593,16 @@ static int spinand_lock_block(struct spinand_device *spinand, u8 lock) }

static int spinand_read_page(struct spinand_device *spinand, - const struct nand_page_io_req *req, - bool ecc_enabled) + const struct nand_page_io_req *req) { + struct nand_device *nand = spinand_to_nand(spinand); u8 status; int ret;

+ ret = nand_ecc_prepare_io_req(nand, (struct nand_page_io_req *)req); + if (ret) + return ret; + ret = spinand_load_page_op(spinand, req); if (ret) return ret; @@ -515,22 +614,26 @@ static int spinand_read_page(struct spinand_device *spinand, if (ret < 0) return ret;

+ spinand_ondie_ecc_save_status(nand, status); + ret = spinand_read_from_cache_op(spinand, req); if (ret) return ret;

- if (!ecc_enabled) - return 0; - - return spinand_check_ecc_status(spinand, status); + return nand_ecc_finish_io_req(nand, (struct nand_page_io_req *)req); }

static int spinand_write_page(struct spinand_device *spinand, const struct nand_page_io_req *req) { + struct nand_device *nand = spinand_to_nand(spinand); u8 status; int ret;

+ ret = nand_ecc_prepare_io_req(nand, (struct nand_page_io_req *)req); + if (ret) + return ret; + ret = spinand_write_enable_op(spinand); if (ret) return ret; @@ -550,7 +653,7 @@ static int spinand_write_page(struct spinand_device *spinand, if (!ret && (status & STATUS_PROG_FAILED)) return -EIO;

- return ret; + return nand_ecc_finish_io_req(nand, (struct nand_page_io_req *)req); }

static int spinand_mtd_read(struct mtd_info *mtd, loff_t from, @@ -580,21 +683,14 @@ static int spinand_mtd_read(struct mtd_info *mtd, loff_t from, if (ret) break;

- ret = spinand_ecc_enable(spinand, !disable_ecc); - if (ret) - break; - - ret = spinand_read_page(spinand, &iter.req, !disable_ecc); + ret = spinand_read_page(spinand, &iter.req); if (ret < 0 && ret != -EBADMSG) break;

- if (ret == -EBADMSG) { + if (ret == -EBADMSG) ecc_failed = true; - mtd->ecc_stats.failed++; - } else { - mtd->ecc_stats.corrected += ret; + else max_bitflips = max_t(unsigned int, max_bitflips, ret); - }

ret = 0; ops->retlen += iter.req.datalen; @@ -635,10 +731,6 @@ static int spinand_mtd_write(struct mtd_info *mtd, loff_t to, if (ret) break;

- ret = spinand_ecc_enable(spinand, !disable_ecc); - if (ret) - break; - ret = spinand_write_page(spinand, &iter.req); if (ret) break; @@ -667,7 +759,7 @@ static bool spinand_isbad(struct nand_device *nand, const struct nand_pos *pos) };

spinand_select_target(spinand, pos->target); - spinand_read_page(spinand, &req, false); + spinand_read_page(spinand, &req); if (marker[0] != 0xff || marker[1] != 0xff) return true;

@@ -835,6 +927,36 @@ static int spinand_create_dirmap(struct spinand_device *spinand,

spinand->dirmaps[plane].rdesc = desc;

+ if (nand->ecc.engine->integration != NAND_ECC_ENGINE_INTEGRATION_PIPELINED) { + spinand->dirmaps[plane].wdesc_ecc = spinand->dirmaps[plane].wdesc; + spinand->dirmaps[plane].rdesc_ecc = spinand->dirmaps[plane].rdesc; + + return 0; + } + + info.op_tmpl = *spinand->op_templates.update_cache; + info.op_tmpl.data.ecc = true; + desc = spi_mem_dirmap_create(spinand->slave, &info); + if (IS_ERR(desc)) { + spi_mem_dirmap_destroy(spinand->dirmaps[plane].wdesc); + spi_mem_dirmap_destroy(spinand->dirmaps[plane].rdesc); + return PTR_ERR(desc); + } + + spinand->dirmaps[plane].wdesc_ecc = desc; + + info.op_tmpl = *spinand->op_templates.read_cache; + info.op_tmpl.data.ecc = true; + desc = spi_mem_dirmap_create(spinand->slave, &info); + if (IS_ERR(desc)) { + spi_mem_dirmap_destroy(spinand->dirmaps[plane].wdesc); + spi_mem_dirmap_destroy(spinand->dirmaps[plane].rdesc); + spi_mem_dirmap_destroy(spinand->dirmaps[plane].wdesc_ecc); + return PTR_ERR(desc); + } + + spinand->dirmaps[plane].rdesc_ecc = desc; + return 0; }

@@ -1019,7 +1141,7 @@ int spinand_match_and_init(struct spinand_device *spinand, continue;

nand->memorg = table[i].memorg; - nand->eccreq = table[i].eccreq; + nanddev_set_ecc_requirements(nand, &table[i].eccreq); spinand->eccinfo = table[i].eccinfo; spinand->flags = table[i].flags; spinand->id.len = 1 + table[i].devid.len; @@ -1171,6 +1293,15 @@ static int spinand_init(struct spinand_device *spinand) if (ret) goto err_manuf_cleanup;

+ /* SPI-NAND default ECC engine is on-die */ + nand->ecc.defaults.engine_type = NAND_ECC_ENGINE_TYPE_ON_DIE; + nand->ecc.ondie_engine = &spinand_ondie_ecc_engine; + + spinand_ecc_enable(spinand, false); + ret = nanddev_ecc_engine_init(nand); + if (ret) + goto err_cleanup_nanddev; + mtd->_read_oob = spinand_mtd_read; mtd->_write_oob = spinand_mtd_write; mtd->_block_isbad = spinand_mtd_block_isbad; @@ -1178,27 +1309,31 @@ static int spinand_init(struct spinand_device *spinand) mtd->_block_isreserved = spinand_mtd_block_isreserved; mtd->_erase = spinand_mtd_erase;

- if (spinand->eccinfo.ooblayout) - mtd_set_ooblayout(mtd, spinand->eccinfo.ooblayout); - else - mtd_set_ooblayout(mtd, &spinand_noecc_ooblayout); - - ret = mtd_ooblayout_count_freebytes(mtd); - if (ret < 0) - goto err_cleanup_nanddev; + if (nand->ecc.engine) { + ret = mtd_ooblayout_count_freebytes(mtd); + if (ret < 0) + goto err_cleanup_ecc_engine; + }

mtd->oobavail = ret;

+ /* Propagate ECC information to mtd_info */ + mtd->ecc_strength = nanddev_get_ecc_conf(nand)->strength; + mtd->ecc_step_size = nanddev_get_ecc_conf(nand)->step_size; + ret = spinand_create_dirmaps(spinand); if (ret) { dev_err(dev, "Failed to create direct mappings for read/write operations (err = %d)\n", ret); - goto err_cleanup_nanddev; + goto err_cleanup_ecc_engine; }

return 0;

+err_cleanup_ecc_engine: + nanddev_ecc_engine_cleanup(nand); + err_cleanup_nanddev: nanddev_cleanup(nand);

diff --git a/drivers/mtd/nand/spi/foresee.c b/drivers/mtd/nand/spi/foresee.c index 7d141cdd658..6229c959b2c 100644 --- a/drivers/mtd/nand/spi/foresee.c +++ b/drivers/mtd/nand/spi/foresee.c @@ -60,7 +60,7 @@ static int f35sqa002g_ecc_get_status(struct spinand_device *spinand, u8 status) return 0;

case STATUS_ECC_HAS_BITFLIPS: - return nand->eccreq.strength; + return nanddev_get_ecc_conf(nand)->strength;

default: break; diff --git a/drivers/mtd/nand/spi/macronix.c b/drivers/mtd/nand/spi/macronix.c index 3d4a7f0c3cb..c2a7aa2da96 100644 --- a/drivers/mtd/nand/spi/macronix.c +++ b/drivers/mtd/nand/spi/macronix.c @@ -87,11 +87,12 @@ static int mx35lf1ge4ab_ecc_get_status(struct spinand_device *spinand, * data around if it's not necessary. */ if (mx35lf1ge4ab_get_eccsr(spinand, spinand->scratchbuf)) - return nand->eccreq.strength; + return nanddev_get_ecc_conf(nand)->strength;

eccsr = *spinand->scratchbuf; - if (WARN_ON(eccsr > nand->eccreq.strength || !eccsr)) - return nand->eccreq.strength; + if (WARN_ON(eccsr > nanddev_get_ecc_conf(nand)->strength || + !eccsr)) + return nanddev_get_ecc_conf(nand)->strength;

return eccsr;

diff --git a/drivers/mtd/nand/spi/micron.c b/drivers/mtd/nand/spi/micron.c index b538213ed8e..01c177facfb 100644 --- a/drivers/mtd/nand/spi/micron.c +++ b/drivers/mtd/nand/spi/micron.c @@ -14,7 +14,7 @@

#define SPINAND_MFR_MICRON 0x2c

-#define MICRON_STATUS_ECC_MASK GENMASK(7, 4) +#define MICRON_STATUS_ECC_MASK GENMASK(6, 4) #define MICRON_STATUS_ECC_NO_BITFLIPS (0 << 4) #define MICRON_STATUS_ECC_1TO3_BITFLIPS (1 << 4) #define MICRON_STATUS_ECC_4TO6_BITFLIPS (3 << 4) diff --git a/drivers/mtd/nand/spi/toshiba.c b/drivers/mtd/nand/spi/toshiba.c index ad48b1c7c8a..bf7da57de13 100644 --- a/drivers/mtd/nand/spi/toshiba.c +++ b/drivers/mtd/nand/spi/toshiba.c @@ -76,7 +76,7 @@ static int tx58cxgxsxraix_ecc_get_status(struct spinand_device *spinand, { struct nand_device *nand = spinand_to_nand(spinand); u8 mbf = 0; - struct spi_mem_op op = SPINAND_GET_FEATURE_OP(0x30, &mbf); + struct spi_mem_op op = SPINAND_GET_FEATURE_OP(0x30, spinand->scratchbuf);

switch (status & STATUS_ECC_MASK) { case STATUS_ECC_NO_BITFLIPS: @@ -93,12 +93,12 @@ static int tx58cxgxsxraix_ecc_get_status(struct spinand_device *spinand, * data around if it's not necessary. */ if (spi_mem_exec_op(spinand->slave, &op)) - return nand->eccreq.strength; + return nanddev_get_ecc_conf(nand)->strength;

- mbf >>= 4; + mbf = *(spinand->scratchbuf) >> 4;

- if (WARN_ON(mbf > nand->eccreq.strength || !mbf)) - return nand->eccreq.strength; + if (WARN_ON(mbf > nanddev_get_ecc_conf(nand)->strength || !mbf)) + return nanddev_get_ecc_conf(nand)->strength;

return mbf;

diff --git a/drivers/mtd/nand/spi/winbond.c b/drivers/mtd/nand/spi/winbond.c index c62096dc2e6..d7dc1c86494 100644 --- a/drivers/mtd/nand/spi/winbond.c +++ b/drivers/mtd/nand/spi/winbond.c @@ -113,7 +113,7 @@ static int w25n02kv_ecc_get_status(struct spinand_device *spinand, { struct nand_device *nand = spinand_to_nand(spinand); u8 mbf = 0; - struct spi_mem_op op = SPINAND_GET_FEATURE_OP(0x30, &mbf); + struct spi_mem_op op = SPINAND_GET_FEATURE_OP(0x30, spinand->scratchbuf);

switch (status & STATUS_ECC_MASK) { case STATUS_ECC_NO_BITFLIPS: @@ -130,12 +130,12 @@ static int w25n02kv_ecc_get_status(struct spinand_device *spinand, * data around if it's not necessary. */ if (spi_mem_exec_op(spinand->slave, &op)) - return nand->eccreq.strength; + return nanddev_get_ecc_conf(nand)->strength;

- mbf >>= 4; + mbf = *(spinand->scratchbuf) >> 4;

- if (WARN_ON(mbf > nand->eccreq.strength || !mbf)) - return nand->eccreq.strength; + if (WARN_ON(mbf > nanddev_get_ecc_conf(nand)->strength || !mbf)) + return nanddev_get_ecc_conf(nand)->strength;

return mbf;

diff --git a/include/linux/mtd/nand.h b/include/linux/mtd/nand.h index 0afdaed5715..18b9cf276ac 100644 --- a/include/linux/mtd/nand.h +++ b/include/linux/mtd/nand.h @@ -12,6 +12,8 @@

#include <linux/mtd/mtd.h>

+struct nand_device; + /** * struct nand_memory_organization - Memory organization structure * @bits_per_cell: number of bits per NAND cell @@ -126,17 +128,72 @@ struct nand_page_io_req { };

/** - * struct nand_ecc_req - NAND ECC requirements + * enum nand_ecc_engine_type - NAND ECC engine type + * @NAND_ECC_ENGINE_TYPE_INVALID: Invalid value + * @NAND_ECC_ENGINE_TYPE_NONE: No ECC correction + * @NAND_ECC_ENGINE_TYPE_SOFT: Software ECC correction + * @NAND_ECC_ENGINE_TYPE_ON_HOST: On host hardware ECC correction + * @NAND_ECC_ENGINE_TYPE_ON_DIE: On chip hardware ECC correction + */ +enum nand_ecc_engine_type { + NAND_ECC_ENGINE_TYPE_INVALID, + NAND_ECC_ENGINE_TYPE_NONE, + NAND_ECC_ENGINE_TYPE_SOFT, + NAND_ECC_ENGINE_TYPE_ON_HOST, + NAND_ECC_ENGINE_TYPE_ON_DIE, +}; + +/** + * enum nand_ecc_placement - NAND ECC bytes placement + * @NAND_ECC_PLACEMENT_UNKNOWN: The actual position of the ECC bytes is unknown + * @NAND_ECC_PLACEMENT_OOB: The ECC bytes are located in the OOB area + * @NAND_ECC_PLACEMENT_INTERLEAVED: Syndrome layout, there are ECC bytes + * interleaved with regular data in the main + * area + */ +enum nand_ecc_placement { + NAND_ECC_PLACEMENT_UNKNOWN, + NAND_ECC_PLACEMENT_OOB, + NAND_ECC_PLACEMENT_INTERLEAVED, +}; + +/** + * enum nand_ecc_algo - NAND ECC algorithm + * @NAND_ECC_ALGO_UNKNOWN: Unknown algorithm + * @NAND_ECC_ALGO_HAMMING: Hamming algorithm + * @NAND_ECC_ALGO_BCH: Bose-Chaudhuri-Hocquenghem algorithm + * @NAND_ECC_ALGO_RS: Reed-Solomon algorithm + */ +enum nand_ecc_algo { + NAND_ECC_ALGO_UNKNOWN, + NAND_ECC_ALGO_HAMMING, + NAND_ECC_ALGO_BCH, + NAND_ECC_ALGO_RS, +}; + +/** + * struct nand_ecc_props - NAND ECC properties + * @engine_type: ECC engine type + * @placement: OOB placement (if relevant) + * @algo: ECC algorithm (if relevant) * @strength: ECC strength - * @step_size: ECC step/block size + * @step_size: Number of bytes per step + * @flags: Misc properties */ -struct nand_ecc_req { +struct nand_ecc_props { + enum nand_ecc_engine_type engine_type; + enum nand_ecc_placement placement; + enum nand_ecc_algo algo; unsigned int strength; unsigned int step_size; + unsigned int flags; };

#define NAND_ECCREQ(str, stp) { .strength = (str), .step_size = (stp) }

+/* NAND ECC misc flags */ +#define NAND_ECC_MAXIMIZE_STRENGTH BIT(0) + /** * struct nand_bbt - bad block table object * @cache: in memory BBT cache @@ -145,8 +202,6 @@ struct nand_bbt { unsigned long *cache; };

-struct nand_device; - /** * struct nand_ops - NAND operations * @erase: erase a specific block. No need to check if the block is bad before @@ -169,11 +224,130 @@ struct nand_ops { bool (*isbad)(struct nand_device *nand, const struct nand_pos *pos); };

+/** + * struct nand_ecc_context - Context for the ECC engine + * @conf: basic ECC engine parameters + * @nsteps: number of ECC steps + * @total: total number of bytes used for storing ECC codes, this is used by + * generic OOB layouts + * @priv: ECC engine driver private data + */ +struct nand_ecc_context { + struct nand_ecc_props conf; + unsigned int nsteps; + unsigned int total; + void *priv; +}; + +/** + * struct nand_ecc_engine_ops - ECC engine operations + * @init_ctx: given a desired user configuration for the pointed NAND device, + * requests the ECC engine driver to setup a configuration with + * values it supports. + * @cleanup_ctx: clean the context initialized by @init_ctx. + * @prepare_io_req: is called before reading/writing a page to prepare the I/O + * request to be performed with ECC correction. + * @finish_io_req: is called after reading/writing a page to terminate the I/O + * request and ensure proper ECC correction. + */ +struct nand_ecc_engine_ops { + int (*init_ctx)(struct nand_device *nand); + void (*cleanup_ctx)(struct nand_device *nand); + int (*prepare_io_req)(struct nand_device *nand, + struct nand_page_io_req *req); + int (*finish_io_req)(struct nand_device *nand, + struct nand_page_io_req *req); +}; + +/** + * enum nand_ecc_engine_integration - How the NAND ECC engine is integrated + * @NAND_ECC_ENGINE_INTEGRATION_INVALID: Invalid value + * @NAND_ECC_ENGINE_INTEGRATION_PIPELINED: Pipelined engine, performs on-the-fly + * correction, does not need to copy + * data around + * @NAND_ECC_ENGINE_INTEGRATION_EXTERNAL: External engine, needs to bring the + * data into its own area before use + */ +enum nand_ecc_engine_integration { + NAND_ECC_ENGINE_INTEGRATION_INVALID, + NAND_ECC_ENGINE_INTEGRATION_PIPELINED, + NAND_ECC_ENGINE_INTEGRATION_EXTERNAL, +}; + +/** + * struct nand_ecc_engine - ECC engine abstraction for NAND devices + * @dev: Host device + * @node: Private field for registration time + * @ops: ECC engine operations + * @integration: How the engine is integrated with the host + * (only relevant on %NAND_ECC_ENGINE_TYPE_ON_HOST engines) + * @priv: Private data + */ +struct nand_ecc_engine { + struct device *dev; + struct list_head node; + struct nand_ecc_engine_ops *ops; + enum nand_ecc_engine_integration integration; + void *priv; +}; + +void of_get_nand_ecc_user_config(struct nand_device *nand); +int nand_ecc_init_ctx(struct nand_device *nand); +void nand_ecc_cleanup_ctx(struct nand_device *nand); +int nand_ecc_prepare_io_req(struct nand_device *nand, + struct nand_page_io_req *req); +int nand_ecc_finish_io_req(struct nand_device *nand, + struct nand_page_io_req *req); +bool nand_ecc_is_strong_enough(struct nand_device *nand); + +struct nand_ecc_engine *nand_ecc_get_sw_engine(struct nand_device *nand); +struct nand_ecc_engine *nand_ecc_get_on_die_hw_engine(struct nand_device *nand); +struct nand_ecc_engine *nand_ecc_get_on_host_hw_engine(struct nand_device *nand); +void nand_ecc_put_on_host_hw_engine(struct nand_device *nand); +struct device *nand_ecc_get_engine_dev(struct device *host); + +#if defined(CONFIG_MTD_NAND_ECC_SW_HAMMING) +struct nand_ecc_engine *nand_ecc_sw_hamming_get_engine(void); +#else +static inline struct nand_ecc_engine *nand_ecc_sw_hamming_get_engine(void) +{ + return NULL; +} +#endif + +#if defined(CONFIG_MTD_NAND_ECC_SW_BCH) +struct nand_ecc_engine *nand_ecc_sw_bch_get_engine(void); +#else +static inline struct nand_ecc_engine *nand_ecc_sw_bch_get_engine(void) +{ + return NULL; +} +#endif + +/** + * struct nand_ecc - Information relative to the ECC + * @defaults: Default values, depend on the underlying subsystem + * @requirements: ECC requirements from the NAND chip perspective + * @user_conf: User desires in terms of ECC parameters + * @ctx: ECC context for the ECC engine, derived from the device @requirements + * the @user_conf and the @defaults + * @ondie_engine: On-die ECC engine reference, if any + * @engine: ECC engine actually bound + */ +struct nand_ecc { + struct nand_ecc_props defaults; + struct nand_ecc_props requirements; + struct nand_ecc_props user_conf; + struct nand_ecc_context ctx; + struct nand_ecc_engine *ondie_engine; + struct nand_ecc_engine *engine; +}; + /** * struct nand_device - NAND device * @mtd: MTD instance attached to the NAND device * @memorg: memory layout - * @eccreq: ECC requirements + * @ecc: NAND ECC object attached to the NAND device * @rowconv: position to row address converter * @bbt: bad block table info * @ops: NAND operations attached to the NAND device @@ -181,8 +355,8 @@ struct nand_ops { * Generic NAND object. Specialized NAND layers (raw NAND, SPI NAND, OneNAND) * should declare their own NAND object embedding a nand_device struct (that's * how inheritance is done). - * struct_nand_device->memorg and struct_nand_device->eccreq should be filled - * at device detection time to reflect the NAND device + * struct_nand_device->memorg and struct_nand_device->ecc.requirements should + * be filled at device detection time to reflect the NAND device * capabilities/requirements. Once this is done nanddev_init() can be called. * It will take care of converting NAND information into MTD ones, which means * the specialized NAND layers should never manually tweak @@ -191,7 +365,7 @@ struct nand_ops { struct nand_device { struct mtd_info *mtd; struct nand_memory_organization memorg; - struct nand_ecc_req eccreq; + struct nand_ecc ecc; struct nand_row_converter rowconv; struct nand_bbt bbt; const struct nand_ops *ops; @@ -332,7 +506,7 @@ static inline unsigned int nanddev_ntargets(const struct nand_device *nand) }

/** - * nanddev_neraseblocks() - Get the total number of erasablocks + * nanddev_neraseblocks() - Get the total number of eraseblocks * @nand: NAND device * * Return: the total number of eraseblocks exposed by @nand. @@ -370,6 +544,60 @@ nanddev_get_memorg(struct nand_device *nand) return &nand->memorg; }

+/** + * nanddev_get_ecc_conf() - Extract the ECC configuration from a NAND device + * @nand: NAND device + */ +static inline const struct nand_ecc_props * +nanddev_get_ecc_conf(struct nand_device *nand) +{ + return &nand->ecc.ctx.conf; +} + +/** + * nanddev_get_ecc_nsteps() - Extract the number of ECC steps + * @nand: NAND device + */ +static inline unsigned int +nanddev_get_ecc_nsteps(struct nand_device *nand) +{ + return nand->ecc.ctx.nsteps; +} + +/** + * nanddev_get_ecc_bytes_per_step() - Extract the number of ECC bytes per step + * @nand: NAND device + */ +static inline unsigned int +nanddev_get_ecc_bytes_per_step(struct nand_device *nand) +{ + return nand->ecc.ctx.total / nand->ecc.ctx.nsteps; +} + +/** + * nanddev_get_ecc_requirements() - Extract the ECC requirements from a NAND + * device + * @nand: NAND device + */ +static inline const struct nand_ecc_props * +nanddev_get_ecc_requirements(struct nand_device *nand) +{ + return &nand->ecc.requirements; +} + +/** + * nanddev_set_ecc_requirements() - Assign the ECC requirements of a NAND + * device + * @nand: NAND device + * @reqs: Requirements + */ +static inline void +nanddev_set_ecc_requirements(struct nand_device *nand, + const struct nand_ecc_props *reqs) +{ + nand->ecc.requirements = *reqs; +} + int nanddev_init(struct nand_device *nand, const struct nand_ops *ops, struct module *owner); void nanddev_cleanup(struct nand_device *nand); @@ -598,7 +826,7 @@ static inline void nanddev_pos_next_eraseblock(struct nand_device *nand, }

/** - * nanddev_pos_next_eraseblock() - Move a position to the next page + * nanddev_pos_next_page() - Move a position to the next page * @nand: NAND device * @pos: the position to update * @@ -708,8 +936,18 @@ static inline bool nanddev_io_iter_end(struct nand_device *nand,

bool nanddev_isbad(struct nand_device *nand, const struct nand_pos *pos); bool nanddev_isreserved(struct nand_device *nand, const struct nand_pos *pos); +int nanddev_erase(struct nand_device *nand, const struct nand_pos *pos); int nanddev_markbad(struct nand_device *nand, const struct nand_pos *pos);

+/* ECC related functions */ +int nanddev_ecc_engine_init(struct nand_device *nand); +void nanddev_ecc_engine_cleanup(struct nand_device *nand); + +static inline void *nand_to_ecc_ctx(struct nand_device *nand) +{ + return nand->ecc.ctx.priv; +} + /* BBT related functions */ enum nand_bbt_block_status { NAND_BBT_BLOCK_STATUS_UNKNOWN, @@ -760,5 +998,6 @@ static inline bool nanddev_bbt_is_initialized(struct nand_device *nand)

/* MTD -> NAND helper functions. */ int nanddev_mtd_erase(struct mtd_info *mtd, struct erase_info *einfo); +int nanddev_mtd_max_bad_blocks(struct mtd_info *mtd, loff_t offs, size_t len);

#endif /* __LINUX_MTD_NAND_H */ diff --git a/include/linux/mtd/spinand.h b/include/linux/mtd/spinand.h index 81a7b0dbbb2..3bcdbffc34a 100644 --- a/include/linux/mtd/spinand.h +++ b/include/linux/mtd/spinand.h @@ -319,6 +319,15 @@ struct spinand_ecc_info { #define SPINAND_HAS_QE_BIT BIT(0) #define SPINAND_HAS_CR_FEAT_BIT BIT(1)

+/** + * struct spinand_ondie_ecc_conf - private SPI-NAND on-die ECC engine structure + * @status: status of the last wait operation that will be used in case + * ->get_status() is not populated by the spinand device. + */ +struct spinand_ondie_ecc_conf { + u8 status; +}; + /** * struct spinand_info - Structure used to describe SPI NAND chips * @model: model name @@ -342,7 +351,7 @@ struct spinand_info { struct spinand_devid devid; u32 flags; struct nand_memory_organization memorg; - struct nand_ecc_req eccreq; + struct nand_ecc_props eccreq; struct spinand_ecc_info eccinfo; struct { const struct spinand_op_variants *read_cache; @@ -391,6 +400,8 @@ struct spinand_info { struct spinand_dirmap { struct spi_mem_dirmap_desc *wdesc; struct spi_mem_dirmap_desc *rdesc; + struct spi_mem_dirmap_desc *wdesc_ecc; + struct spi_mem_dirmap_desc *rdesc_ecc; };

/** diff --git a/include/spi-mem.h b/include/spi-mem.h index 3c8e95b6f53..82dbe21fd5a 100644 --- a/include/spi-mem.h +++ b/include/spi-mem.h @@ -91,6 +91,7 @@ enum spi_mem_data_dir { * @dummy.dtr: whether the dummy bytes should be sent in DTR mode or not * @data.buswidth: number of IO lanes used to send/receive the data * @data.dtr: whether the data should be sent in DTR mode or not + * @data.ecc: whether error correction is required or not * @data.dir: direction of the transfer * @data.buf.in: input buffer * @data.buf.out: output buffer @@ -119,6 +120,7 @@ struct spi_mem_op { struct { u8 buswidth; u8 dtr : 1; + u8 ecc : 1; enum spi_mem_data_dir dir; unsigned int nbytes; /* buf.{in,out} must be DMA-able. */